@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,233 lines 72.2 KB
Raw
1 /*
2 * QEMU PowerPC XIVE2 interrupt controller model (POWER10)
3 *
4 * Copyright (c) 2019-2024, IBM Corporation..
5 *
6 * SPDX-License-Identifier: GPL-2.0-or-later
7 */
8
9 #include "qemu/osdep.h"
10 #include "qemu/log.h"
11 #include "qemu/module.h"
12 #include "qapi/error.h"
13 #include "target/ppc/cpu.h"
14 #include "system/cpus.h"
15 #include "system/dma.h"
16 #include "system/physmem.h"
17 #include "hw/core/qdev-properties.h"
18 #include "hw/ppc/xive.h"
19 #include "hw/ppc/xive2.h"
20 #include "hw/ppc/xive2_regs.h"
21 #include "exec/cpu-common.h"
22 #include "trace.h"
23
24 static void xive2_router_end_notify(Xive2Router *xrtr, uint8_t end_blk,
25 uint32_t end_idx, uint32_t end_data,
26 bool redistribute);
27
28 static int xive2_tctx_get_nvp_indexes(XiveTCTX *tctx, uint8_t ring,
29 uint8_t *nvp_blk, uint32_t *nvp_idx);
30
31 uint32_t xive2_router_get_config(Xive2Router *xrtr)
32 {
33 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
34
35 return xrc->get_config(xrtr);
36 }
37
38 static int xive2_router_get_block_id(Xive2Router *xrtr)
39 {
40 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
41
42 return xrc->get_block_id(xrtr);
43 }
44
45 static uint64_t xive2_nvp_reporting_addr(Xive2Nvp *nvp)
46 {
47 uint64_t cache_addr;
48
49 cache_addr = xive_get_field32(NVP2_W6_REPORTING_LINE, nvp->w6) << 24 |
50 xive_get_field32(NVP2_W7_REPORTING_LINE, nvp->w7);
51 cache_addr <<= 8; /* aligned on a cache line pair */
52 return cache_addr;
53 }
54
55 static uint32_t xive2_nvgc_get_backlog(Xive2Nvgc *nvgc, uint8_t priority)
56 {
57 uint32_t val = 0;
58 uint8_t *ptr, i;
59
60 if (priority > 7) {
61 return 0;
62 }
63
64 /*
65 * The per-priority backlog counters are 24-bit and the structure
66 * is stored in big endian. NVGC is 32-bytes long, so 24-bytes from
67 * w2, which fits 8 priorities * 24-bits per priority.
68 */
69 ptr = (uint8_t *)&nvgc->w2 + priority * 3;
70 for (i = 0; i < 3; i++, ptr++) {
71 val = (val << 8) + *ptr;
72 }
73 return val;
74 }
75
76 static void xive2_nvgc_set_backlog(Xive2Nvgc *nvgc, uint8_t priority,
77 uint32_t val)
78 {
79 uint8_t *ptr, i;
80 uint32_t shift;
81
82 if (priority > 7) {
83 return;
84 }
85
86 if (val > 0xFFFFFF) {
87 val = 0xFFFFFF;
88 }
89 /*
90 * The per-priority backlog counters are 24-bit and the structure
91 * is stored in big endian
92 */
93 ptr = (uint8_t *)&nvgc->w2 + priority * 3;
94 for (i = 0; i < 3; i++, ptr++) {
95 shift = 8 * (2 - i);
96 *ptr = (val >> shift) & 0xFF;
97 }
98 }
99
100 static uint32_t xive2_nvgc_get_idx(uint32_t nvp_idx, uint8_t group)
101 {
102 uint32_t nvgc_idx;
103
104 if (group > 0) {
105 nvgc_idx = (nvp_idx & (0xffffffffULL << group)) |
106 ((1 << (group - 1)) - 1);
107 } else {
108 nvgc_idx = nvp_idx;
109 }
110
111 return nvgc_idx;
112 }
113
114 static uint8_t xive2_nvgc_get_blk(uint8_t nvp_blk, uint8_t crowd)
115 {
116 uint8_t nvgc_blk;
117
118 if (crowd > 0) {
119 crowd = (crowd == 3) ? 4 : crowd;
120 nvgc_blk = (nvp_blk & (0xffffffffULL << crowd)) |
121 ((1 << (crowd - 1)) - 1);
122 } else {
123 nvgc_blk = nvp_blk;
124 }
125
126 return nvgc_blk;
127 }
128
129 uint64_t xive2_presenter_nvgc_backlog_op(XivePresenter *xptr,
130 bool crowd,
131 uint8_t blk, uint32_t idx,
132 uint16_t offset, uint16_t val)
133 {
134 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
135 uint8_t priority = GETFIELD(NVx_BACKLOG_PRIO, offset);
136 uint8_t op = GETFIELD(NVx_BACKLOG_OP, offset);
137 Xive2Nvgc nvgc;
138 uint32_t count, old_count;
139
140 if (xive2_router_get_nvgc(xrtr, crowd, blk, idx, &nvgc)) {
141 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No %s %x/%x\n",
142 crowd ? "NVC" : "NVG", blk, idx);
143 return -1;
144 }
145 if (!xive2_nvgc_is_valid(&nvgc)) {
146 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid NVG %x/%x\n", blk, idx);
147 return -1;
148 }
149
150 old_count = xive2_nvgc_get_backlog(&nvgc, priority);
151 count = old_count;
152 /*
153 * op:
154 * 0b00 => increment
155 * 0b01 => decrement
156 * 0b1- => read
157 */
158 if (op == 0b00 || op == 0b01) {
159 if (op == 0b00) {
160 count += val;
161 } else {
162 if (count > val) {
163 count -= val;
164 } else {
165 count = 0;
166 }
167 }
168 xive2_nvgc_set_backlog(&nvgc, priority, count);
169 xive2_router_write_nvgc(xrtr, crowd, blk, idx, &nvgc);
170 }
171 trace_xive_nvgc_backlog_op(crowd, blk, idx, op, priority, old_count);
172 return old_count;
173 }
174
175 uint64_t xive2_presenter_nvp_backlog_op(XivePresenter *xptr,
176 uint8_t blk, uint32_t idx,
177 uint16_t offset)
178 {
179 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
180 uint8_t priority = GETFIELD(NVx_BACKLOG_PRIO, offset);
181 uint8_t op = GETFIELD(NVx_BACKLOG_OP, offset);
182 Xive2Nvp nvp;
183 uint8_t ipb, old_ipb, rc;
184
185 if (xive2_router_get_nvp(xrtr, blk, idx, &nvp)) {
186 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n", blk, idx);
187 return -1;
188 }
189 if (!xive2_nvp_is_valid(&nvp)) {
190 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid NVP %x/%x\n", blk, idx);
191 return -1;
192 }
193
194 old_ipb = xive_get_field32(NVP2_W2_IPB, nvp.w2);
195 ipb = old_ipb;
196 /*
197 * op:
198 * 0b00 => set priority bit
199 * 0b01 => reset priority bit
200 * 0b1- => read
201 */
202 if (op == 0b00 || op == 0b01) {
203 if (op == 0b00) {
204 ipb |= xive_priority_to_ipb(priority);
205 } else {
206 ipb &= ~xive_priority_to_ipb(priority);
207 }
208 nvp.w2 = xive_set_field32(NVP2_W2_IPB, nvp.w2, ipb);
209 xive2_router_write_nvp(xrtr, blk, idx, &nvp, 2);
210 }
211 rc = !!(old_ipb & xive_priority_to_ipb(priority));
212 trace_xive_nvp_backlog_op(blk, idx, op, priority, rc);
213 return rc;
214 }
215
216 void xive2_eas_pic_print_info(Xive2Eas *eas, uint32_t lisn, GString *buf)
217 {
218 if (!xive2_eas_is_valid(eas)) {
219 return;
220 }
221
222 g_string_append_printf(buf, " %08x %s end:%02x/%04x data:%08x\n",
223 lisn, xive2_eas_is_masked(eas) ? "M" : " ",
224 (uint8_t) xive_get_field64(EAS2_END_BLOCK, eas->w),
225 (uint32_t) xive_get_field64(EAS2_END_INDEX, eas->w),
226 (uint32_t) xive_get_field64(EAS2_END_DATA, eas->w));
227 }
228
229 #define XIVE2_QSIZE_CHUNK_CL 128
230 #define XIVE2_QSIZE_CHUNK_4k 4096
231 /* Calculate max number of queue entries for an END */
232 static uint32_t xive2_end_get_qentries(Xive2End *end)
233 {
234 uint32_t w3 = end->w3;
235 uint32_t qsize = xive_get_field32(END2_W3_QSIZE, w3);
236 if (xive_get_field32(END2_W3_CL, w3)) {
237 g_assert(qsize <= 4);
238 return (XIVE2_QSIZE_CHUNK_CL << qsize) / sizeof(uint32_t);
239 } else {
240 g_assert(qsize <= 12);
241 return (XIVE2_QSIZE_CHUNK_4k << qsize) / sizeof(uint32_t);
242 }
243 }
244
245 void xive2_end_queue_pic_print_info(Xive2End *end, uint32_t width, GString *buf)
246 {
247 uint64_t qaddr_base = xive2_end_qaddr(end);
248 uint32_t qindex = xive_get_field32(END2_W1_PAGE_OFF, end->w1);
249 uint32_t qentries = xive2_end_get_qentries(end);
250 int i;
251
252 /*
253 * print out the [ (qindex - (width - 1)) .. (qindex + 1)] window
254 */
255 g_string_append_printf(buf, " [ ");
256 qindex = (qindex - (width - 1)) & (qentries - 1);
257 for (i = 0; i < width; i++) {
258 uint64_t qaddr = qaddr_base + (qindex << 2);
259 uint32_t qdata = -1;
260
261 if (dma_memory_read(&address_space_memory, qaddr, &qdata,
262 sizeof(qdata), MEMTXATTRS_UNSPECIFIED)) {
263 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: failed to read EQ @0x%"
264 HWADDR_PRIx "\n", qaddr);
265 return;
266 }
267 g_string_append_printf(buf, "%s%08x ", i == width - 1 ? "^" : "",
268 be32_to_cpu(qdata));
269 qindex = (qindex + 1) & (qentries - 1);
270 }
271 g_string_append_printf(buf, "]");
272 }
273
274 void xive2_end_pic_print_info(Xive2End *end, uint32_t end_idx, GString *buf)
275 {
276 uint64_t qaddr_base = xive2_end_qaddr(end);
277 uint32_t qindex = xive_get_field32(END2_W1_PAGE_OFF, end->w1);
278 uint32_t qgen = xive_get_field32(END2_W1_GENERATION, end->w1);
279 uint32_t qentries = xive2_end_get_qentries(end);
280
281 uint32_t nvx_blk = xive_get_field32(END2_W6_VP_BLOCK, end->w6);
282 uint32_t nvx_idx = xive_get_field32(END2_W6_VP_OFFSET, end->w6);
283 uint8_t priority = xive_get_field32(END2_W7_F0_PRIORITY, end->w7);
284 uint8_t pq;
285
286 if (!xive2_end_is_valid(end)) {
287 return;
288 }
289
290 pq = xive_get_field32(END2_W1_ESn, end->w1);
291
292 g_string_append_printf(buf,
293 " %08x %c%c %c%c%c%c%c%c%c%c%c%c%c %c%c "
294 "prio:%d nvp:%02x/%04x",
295 end_idx,
296 pq & XIVE_ESB_VAL_P ? 'P' : '-',
297 pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
298 xive2_end_is_valid(end) ? 'v' : '-',
299 xive2_end_is_enqueue(end) ? 'q' : '-',
300 xive2_end_is_notify(end) ? 'n' : '-',
301 xive2_end_is_backlog(end) ? 'b' : '-',
302 xive2_end_is_precluded_escalation(end) ? 'p' : '-',
303 xive2_end_is_escalate(end) ? 'e' : '-',
304 xive2_end_is_escalate_end(end) ? 'N' : '-',
305 xive2_end_is_uncond_escalation(end) ? 'u' : '-',
306 xive2_end_is_silent_escalation(end) ? 's' : '-',
307 xive2_end_is_firmware1(end) ? 'f' : '-',
308 xive2_end_is_firmware2(end) ? 'F' : '-',
309 xive2_end_is_ignore(end) ? 'i' : '-',
310 xive2_end_is_crowd(end) ? 'c' : '-',
311 priority, nvx_blk, nvx_idx);
312
313 if (qaddr_base) {
314 g_string_append_printf(buf, " eq:@%08"PRIx64"% 6d/%5d ^%d",
315 qaddr_base, qindex, qentries, qgen);
316 xive2_end_queue_pic_print_info(end, 6, buf);
317 }
318 g_string_append_c(buf, '\n');
319 }
320
321 void xive2_end_eas_pic_print_info(Xive2End *end, uint32_t end_idx,
322 GString *buf)
323 {
324 Xive2Eas *eas = (Xive2Eas *) &end->w4;
325 uint8_t pq;
326
327 if (!xive2_end_is_escalate(end)) {
328 return;
329 }
330
331 pq = xive_get_field32(END2_W1_ESe, end->w1);
332
333 g_string_append_printf(buf, " %08x %c%c %c%c end:%02x/%04x data:%08x\n",
334 end_idx,
335 pq & XIVE_ESB_VAL_P ? 'P' : '-',
336 pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
337 xive2_eas_is_valid(eas) ? 'v' : ' ',
338 xive2_eas_is_masked(eas) ? 'M' : ' ',
339 (uint8_t) xive_get_field64(EAS2_END_BLOCK, eas->w),
340 (uint32_t) xive_get_field64(EAS2_END_INDEX, eas->w),
341 (uint32_t) xive_get_field64(EAS2_END_DATA, eas->w));
342 }
343
344 void xive2_nvp_pic_print_info(Xive2Nvp *nvp, uint32_t nvp_idx, GString *buf)
345 {
346 uint8_t eq_blk = xive_get_field32(NVP2_W5_VP_END_BLOCK, nvp->w5);
347 uint32_t eq_idx = xive_get_field32(NVP2_W5_VP_END_INDEX, nvp->w5);
348 uint64_t cache_line = xive2_nvp_reporting_addr(nvp);
349
350 if (!xive2_nvp_is_valid(nvp)) {
351 return;
352 }
353
354 g_string_append_printf(buf, " %08x end:%02x/%04x IPB:%02x PGoFirst:%02x",
355 nvp_idx, eq_blk, eq_idx,
356 xive_get_field32(NVP2_W2_IPB, nvp->w2),
357 xive_get_field32(NVP2_W0_PGOFIRST, nvp->w0));
358 if (cache_line) {
359 g_string_append_printf(buf, " reporting CL:%016"PRIx64, cache_line);
360 }
361
362 /*
363 * When the NVP is HW controlled, more fields are updated
364 */
365 if (xive2_nvp_is_hw(nvp)) {
366 g_string_append_printf(buf, " CPPR:%02x",
367 xive_get_field32(NVP2_W2_CPPR, nvp->w2));
368 if (xive2_nvp_is_co(nvp)) {
369 g_string_append_printf(buf, " CO:%04x",
370 xive_get_field32(NVP2_W1_CO_THRID, nvp->w1));
371 }
372 }
373 g_string_append_c(buf, '\n');
374 }
375
376 void xive2_nvgc_pic_print_info(Xive2Nvgc *nvgc, uint32_t nvgc_idx, GString *buf)
377 {
378 uint8_t i;
379
380 if (!xive2_nvgc_is_valid(nvgc)) {
381 return;
382 }
383
384 g_string_append_printf(buf, " %08x PGoNext:%02x bklog: ", nvgc_idx,
385 xive_get_field32(NVGC2_W0_PGONEXT, nvgc->w0));
386 for (i = 0; i <= XIVE_PRIORITY_MAX; i++) {
387 g_string_append_printf(buf, "[%d]=0x%x ",
388 i, xive2_nvgc_get_backlog(nvgc, i));
389 }
390 g_string_append_printf(buf, "\n");
391 }
392
393 static void xive2_end_enqueue(Xive2End *end, uint32_t data)
394 {
395 uint64_t qaddr_base = xive2_end_qaddr(end);
396 uint32_t qindex = xive_get_field32(END2_W1_PAGE_OFF, end->w1);
397 uint32_t qgen = xive_get_field32(END2_W1_GENERATION, end->w1);
398
399 uint64_t qaddr = qaddr_base + (qindex << 2);
400 uint32_t qdata = cpu_to_be32((qgen << 31) | (data & 0x7fffffff));
401 uint32_t qentries = xive2_end_get_qentries(end);
402
403 if (dma_memory_write(&address_space_memory, qaddr, &qdata, sizeof(qdata),
404 MEMTXATTRS_UNSPECIFIED)) {
405 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: failed to write END data @0x%"
406 HWADDR_PRIx "\n", qaddr);
407 return;
408 }
409
410 qindex = (qindex + 1) & (qentries - 1);
411 if (qindex == 0) {
412 qgen ^= 1;
413 end->w1 = xive_set_field32(END2_W1_GENERATION, end->w1, qgen);
414
415 /* Set gen flipped to 1, it gets reset on a cache watch operation */
416 end->w1 = xive_set_field32(END2_W1_GEN_FLIPPED, end->w1, 1);
417 }
418 end->w1 = xive_set_field32(END2_W1_PAGE_OFF, end->w1, qindex);
419 }
420
421 static void xive2_pgofnext(uint8_t *nvgc_blk, uint32_t *nvgc_idx,
422 uint8_t next_level)
423 {
424 uint32_t mask, next_idx;
425 uint8_t next_blk;
426
427 /*
428 * Adjust the block and index of a VP for the next group/crowd
429 * size (PGofFirst/PGofNext field in the NVP and NVGC structures).
430 *
431 * The 6-bit group level is split into a 2-bit crowd and 4-bit
432 * group levels. Encoding is similar. However, we don't support
433 * crowd size of 8. So a crowd level of 0b11 is bumped to a crowd
434 * size of 16.
435 */
436 next_blk = NVx_CROWD_LVL(next_level);
437 if (next_blk == 3) {
438 next_blk = 4;
439 }
440 mask = (1 << next_blk) - 1;
441 *nvgc_blk &= ~mask;
442 *nvgc_blk |= mask >> 1;
443
444 next_idx = NVx_GROUP_LVL(next_level);
445 mask = (1 << next_idx) - 1;
446 *nvgc_idx &= ~mask;
447 *nvgc_idx |= mask >> 1;
448 }
449
450 /*
451 * Scan the group chain and return the highest priority and group
452 * level of pending group interrupts.
453 */
454 static uint8_t xive2_presenter_backlog_scan(XivePresenter *xptr,
455 uint8_t nvx_blk, uint32_t nvx_idx,
456 uint8_t first_group,
457 uint8_t *out_level)
458 {
459 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
460 uint32_t nvgc_idx;
461 uint32_t current_level, count;
462 uint8_t nvgc_blk, prio;
463 Xive2Nvgc nvgc;
464
465 for (prio = 0; prio <= XIVE_PRIORITY_MAX; prio++) {
466 current_level = first_group & 0x3F;
467 nvgc_blk = nvx_blk;
468 nvgc_idx = nvx_idx;
469
470 while (current_level) {
471 xive2_pgofnext(&nvgc_blk, &nvgc_idx, current_level);
472
473 if (xive2_router_get_nvgc(xrtr, NVx_CROWD_LVL(current_level),
474 nvgc_blk, nvgc_idx, &nvgc)) {
475 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVGC %x/%x\n",
476 nvgc_blk, nvgc_idx);
477 return 0xFF;
478 }
479 if (!xive2_nvgc_is_valid(&nvgc)) {
480 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid NVGC %x/%x\n",
481 nvgc_blk, nvgc_idx);
482 return 0xFF;
483 }
484
485 count = xive2_nvgc_get_backlog(&nvgc, prio);
486 if (count) {
487 *out_level = current_level;
488 return prio;
489 }
490 current_level = xive_get_field32(NVGC2_W0_PGONEXT, nvgc.w0) & 0x3F;
491 }
492 }
493 return 0xFF;
494 }
495
496 static void xive2_presenter_backlog_decr(XivePresenter *xptr,
497 uint8_t nvx_blk, uint32_t nvx_idx,
498 uint8_t group_prio,
499 uint8_t group_level)
500 {
501 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
502 uint32_t nvgc_idx, count;
503 uint8_t nvgc_blk;
504 Xive2Nvgc nvgc;
505
506 nvgc_blk = nvx_blk;
507 nvgc_idx = nvx_idx;
508 xive2_pgofnext(&nvgc_blk, &nvgc_idx, group_level);
509
510 if (xive2_router_get_nvgc(xrtr, NVx_CROWD_LVL(group_level),
511 nvgc_blk, nvgc_idx, &nvgc)) {
512 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVGC %x/%x\n",
513 nvgc_blk, nvgc_idx);
514 return;
515 }
516 if (!xive2_nvgc_is_valid(&nvgc)) {
517 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid NVGC %x/%x\n",
518 nvgc_blk, nvgc_idx);
519 return;
520 }
521 count = xive2_nvgc_get_backlog(&nvgc, group_prio);
522 if (!count) {
523 return;
524 }
525 xive2_nvgc_set_backlog(&nvgc, group_prio, count - 1);
526 xive2_router_write_nvgc(xrtr, NVx_CROWD_LVL(group_level),
527 nvgc_blk, nvgc_idx, &nvgc);
528 }
529
530 /*
531 * XIVE Thread Interrupt Management Area (TIMA) - Gen2 mode
532 *
533 * TIMA Gen2 VP “save & restore” (S&R) indicated by H bit next to V bit
534 *
535 * - if a context is enabled with the H bit set, the VP context
536 * information is retrieved from the NVP structure (“check out”)
537 * and stored back on a context pull (“check in”), the SW receives
538 * the same context pull information as on P9
539 *
540 * - the H bit cannot be changed while the V bit is set, i.e. a
541 * context cannot be set up in the TIMA and then be “pushed” into
542 * the NVP by changing the H bit while the context is enabled
543 */
544
545 static void xive2_tctx_save_ctx(Xive2Router *xrtr, XiveTCTX *tctx,
546 uint8_t ring,
547 uint8_t nvp_blk, uint32_t nvp_idx)
548 {
549 CPUPPCState *env = &POWERPC_CPU(tctx->cs)->env;
550 uint32_t pir = env->spr_cb[SPR_PIR].default_value;
551 Xive2Nvp nvp;
552 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, ring);
553 uint8_t *regs = &tctx->regs[ring];
554
555 if (xive2_router_get_nvp(xrtr, nvp_blk, nvp_idx, &nvp)) {
556 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n",
557 nvp_blk, nvp_idx);
558 return;
559 }
560
561 if (!xive2_nvp_is_valid(&nvp)) {
562 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
563 nvp_blk, nvp_idx);
564 return;
565 }
566
567 if (!xive2_nvp_is_hw(&nvp)) {
568 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: NVP %x/%x is not HW owned\n",
569 nvp_blk, nvp_idx);
570 return;
571 }
572
573 if (!xive2_nvp_is_co(&nvp)) {
574 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: NVP %x/%x is not checkout\n",
575 nvp_blk, nvp_idx);
576 return;
577 }
578
579 if (xive_get_field32(NVP2_W1_CO_THRID_VALID, nvp.w1) &&
580 xive_get_field32(NVP2_W1_CO_THRID, nvp.w1) != pir) {
581 qemu_log_mask(LOG_GUEST_ERROR,
582 "XIVE: NVP %x/%x invalid checkout Thread %x\n",
583 nvp_blk, nvp_idx, pir);
584 return;
585 }
586
587 nvp.w2 = xive_set_field32(NVP2_W2_IPB, nvp.w2, regs[TM_IPB]);
588
589 if ((nvp.w0 & NVP2_W0_P) || ring != TM_QW2_HV_POOL) {
590 /*
591 * Non-pool contexts always save CPPR (ignore p bit). XXX: Clarify
592 * whether that is the correct behaviour.
593 */
594 nvp.w2 = xive_set_field32(NVP2_W2_CPPR, nvp.w2, sig_regs[TM_CPPR]);
595 }
596 if (nvp.w0 & NVP2_W0_L) {
597 /*
598 * Typically not used. If LSMFB is restored with 0, it will
599 * force a backlog rescan
600 */
601 nvp.w2 = xive_set_field32(NVP2_W2_LSMFB, nvp.w2, regs[TM_LSMFB]);
602 }
603 if (nvp.w0 & NVP2_W0_G) {
604 nvp.w2 = xive_set_field32(NVP2_W2_LGS, nvp.w2, regs[TM_LGS]);
605 }
606 if (nvp.w0 & NVP2_W0_T) {
607 nvp.w2 = xive_set_field32(NVP2_W2_T, nvp.w2, regs[TM_T]);
608 }
609 xive2_router_write_nvp(xrtr, nvp_blk, nvp_idx, &nvp, 2);
610
611 nvp.w1 = xive_set_field32(NVP2_W1_CO, nvp.w1, 0);
612 /* NVP2_W1_CO_THRID_VALID only set once */
613 nvp.w1 = xive_set_field32(NVP2_W1_CO_THRID, nvp.w1, 0xFFFF);
614 xive2_router_write_nvp(xrtr, nvp_blk, nvp_idx, &nvp, 1);
615 }
616
617 /* POOL cam is the same as OS cam encoding */
618 static void xive2_cam_decode(uint32_t cam, uint8_t *nvp_blk,
619 uint32_t *nvp_idx, bool *valid, bool *hw)
620 {
621 *nvp_blk = xive2_nvp_blk(cam);
622 *nvp_idx = xive2_nvp_idx(cam);
623 *valid = !!(cam & TM2_W2_VALID);
624 *hw = !!(cam & TM2_W2_HW);
625 }
626
627 /*
628 * Encode the HW CAM line with 7bit or 8bit thread id. The thread id
629 * width and block id width is configurable at the IC level.
630 *
631 * chipid << 24 | 0000 0000 0000 0000 1 threadid (7Bit)
632 * chipid << 24 | 0000 0000 0000 0001 threadid (8Bit)
633 */
634 static uint32_t xive2_tctx_hw_cam_line(XivePresenter *xptr, XiveTCTX *tctx)
635 {
636 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
637 CPUPPCState *env = &POWERPC_CPU(tctx->cs)->env;
638 uint32_t pir = env->spr_cb[SPR_PIR].default_value;
639 uint8_t blk = xive2_router_get_block_id(xrtr);
640 uint8_t tid_shift =
641 xive2_router_get_config(xrtr) & XIVE2_THREADID_8BITS ? 8 : 7;
642 uint8_t tid_mask = (1 << tid_shift) - 1;
643
644 return xive2_nvp_cam_line(blk, 1 << tid_shift | (pir & tid_mask));
645 }
646
647 static void xive2_redistribute(Xive2Router *xrtr, XiveTCTX *tctx, uint8_t ring)
648 {
649 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, ring);
650 uint8_t nsr = sig_regs[TM_NSR];
651 uint8_t pipr = sig_regs[TM_PIPR];
652 uint8_t crowd = NVx_CROWD_LVL(nsr);
653 uint8_t group = NVx_GROUP_LVL(nsr);
654 uint8_t nvgc_blk, end_blk, nvp_blk;
655 uint32_t nvgc_idx, end_idx, nvp_idx;
656 Xive2Nvgc nvgc;
657 uint8_t prio_limit;
658 uint32_t cfg;
659
660 /* redistribution is only for group/crowd interrupts */
661 if (!xive_nsr_indicates_group_exception(ring, nsr)) {
662 return;
663 }
664
665 /* Don't check return code since ring is expected to be invalidated */
666 xive2_tctx_get_nvp_indexes(tctx, ring, &nvp_blk, &nvp_idx);
667
668 trace_xive_redistribute(tctx->cs->cpu_index, ring, nvp_blk, nvp_idx);
669
670 trace_xive_redistribute(tctx->cs->cpu_index, ring, nvp_blk, nvp_idx);
671 /* convert crowd/group to blk/idx */
672 nvgc_idx = xive2_nvgc_get_idx(nvp_idx, group);
673 nvgc_blk = xive2_nvgc_get_blk(nvp_blk, crowd);
674
675 /* Use blk/idx to retrieve the NVGC */
676 if (xive2_router_get_nvgc(xrtr, crowd, nvgc_blk, nvgc_idx, &nvgc)) {
677 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: no %s %x/%x\n",
678 crowd ? "NVC" : "NVG", nvgc_blk, nvgc_idx);
679 return;
680 }
681
682 /* retrieve the END blk/idx from the NVGC */
683 end_blk = xive_get_field32(NVGC2_W1_END_BLK, nvgc.w1);
684 end_idx = xive_get_field32(NVGC2_W1_END_IDX, nvgc.w1);
685
686 /* determine number of priorities being used */
687 cfg = xive2_router_get_config(xrtr);
688 if (cfg & XIVE2_EN_VP_GRP_PRIORITY) {
689 prio_limit = 1 << GETFIELD(NVGC2_W1_PSIZE, nvgc.w1);
690 } else {
691 prio_limit = 1 << GETFIELD(XIVE2_VP_INT_PRIO, cfg);
692 }
693
694 /* add priority offset to end index */
695 end_idx += pipr % prio_limit;
696
697 /* trigger the group END */
698 xive2_router_end_notify(xrtr, end_blk, end_idx, 0, true);
699
700 /* clear interrupt indication for the context */
701 sig_regs[TM_NSR] = 0;
702 sig_regs[TM_PIPR] = sig_regs[TM_CPPR];
703 xive_tctx_reset_signal(tctx, ring);
704 }
705
706 static void xive2_tctx_process_pending(XiveTCTX *tctx, uint8_t sig_ring);
707
708 static uint64_t xive2_tm_pull_ctx(XivePresenter *xptr, XiveTCTX *tctx,
709 hwaddr offset, unsigned size, uint8_t ring)
710 {
711 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
712 uint32_t target_ringw2 = xive_tctx_word2(&tctx->regs[ring]);
713 uint32_t cam = be32_to_cpu(target_ringw2);
714 uint8_t nvp_blk;
715 uint32_t nvp_idx;
716 uint8_t cur_ring;
717 bool valid;
718 bool do_save;
719 uint8_t nsr;
720
721 xive2_cam_decode(cam, &nvp_blk, &nvp_idx, &valid, &do_save);
722
723 if (xive2_tctx_get_nvp_indexes(tctx, ring, &nvp_blk, &nvp_idx)) {
724 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: pulling invalid NVP %x/%x !?\n",
725 nvp_blk, nvp_idx);
726 }
727
728 /* Invalidate CAM line of requested ring and all lower rings */
729 for (cur_ring = TM_QW0_USER; cur_ring <= ring;
730 cur_ring += XIVE_TM_RING_SIZE) {
731 uint32_t ringw2 = xive_tctx_word2(&tctx->regs[cur_ring]);
732 uint32_t ringw2_new = xive_set_field32(TM2_QW1W2_VO, ringw2, 0);
733 bool is_valid = !!(xive_get_field32(TM2_QW1W2_VO, ringw2));
734 uint8_t *sig_regs;
735
736 memcpy(&tctx->regs[cur_ring + TM_WORD2], &ringw2_new, 4);
737
738 /* Skip the rest for USER or invalid contexts */
739 if ((cur_ring == TM_QW0_USER) || !is_valid) {
740 continue;
741 }
742
743 /* Active group/crowd interrupts need to be redistributed */
744 sig_regs = xive_tctx_signal_regs(tctx, ring);
745 nsr = sig_regs[TM_NSR];
746 if (xive_nsr_indicates_group_exception(cur_ring, nsr)) {
747 /* Ensure ring matches NSR (for HV NSR POOL vs PHYS rings) */
748 if (cur_ring == xive_nsr_exception_ring(cur_ring, nsr)) {
749 xive2_redistribute(xrtr, tctx, cur_ring);
750 }
751 }
752
753 /*
754 * Lower external interrupt line of requested ring and below except for
755 * USER, which doesn't exist.
756 */
757 if (xive_nsr_indicates_exception(cur_ring, nsr)) {
758 if (cur_ring == xive_nsr_exception_ring(cur_ring, nsr)) {
759 xive_tctx_reset_signal(tctx, cur_ring);
760 }
761 }
762 }
763
764 if (ring == TM_QW2_HV_POOL) {
765 /* Re-check phys for interrupts if pool was disabled */
766 nsr = tctx->regs[TM_QW3_HV_PHYS + TM_NSR];
767 if (xive_nsr_indicates_exception(TM_QW3_HV_PHYS, nsr)) {
768 /* Ring must be PHYS because POOL would have been redistributed */
769 g_assert(xive_nsr_exception_ring(TM_QW3_HV_PHYS, nsr) ==
770 TM_QW3_HV_PHYS);
771 } else {
772 xive2_tctx_process_pending(tctx, TM_QW3_HV_PHYS);
773 }
774 }
775
776 if (xive2_router_get_config(xrtr) & XIVE2_VP_SAVE_RESTORE && do_save) {
777 xive2_tctx_save_ctx(xrtr, tctx, ring, nvp_blk, nvp_idx);
778 }
779
780 return target_ringw2;
781 }
782
783 uint64_t xive2_tm_pull_os_ctx(XivePresenter *xptr, XiveTCTX *tctx,
784 hwaddr offset, unsigned size)
785 {
786 return xive2_tm_pull_ctx(xptr, tctx, offset, size, TM_QW1_OS);
787 }
788
789 uint64_t xive2_tm_pull_pool_ctx(XivePresenter *xptr, XiveTCTX *tctx,
790 hwaddr offset, unsigned size)
791 {
792 return xive2_tm_pull_ctx(xptr, tctx, offset, size, TM_QW2_HV_POOL);
793 }
794
795 uint64_t xive2_tm_pull_phys_ctx(XivePresenter *xptr, XiveTCTX *tctx,
796 hwaddr offset, unsigned size)
797 {
798 return xive2_tm_pull_ctx(xptr, tctx, offset, size, TM_QW3_HV_PHYS);
799 }
800
801 #define REPORT_LINE_GEN1_SIZE 16
802
803 static void xive2_tm_report_line_gen1(XiveTCTX *tctx, uint8_t *data,
804 uint8_t size)
805 {
806 uint8_t *regs = tctx->regs;
807
808 g_assert(size == REPORT_LINE_GEN1_SIZE);
809 memset(data, 0, size);
810 /*
811 * See xive architecture for description of what is saved. It is
812 * hand-picked information to fit in 16 bytes.
813 */
814 data[0x0] = regs[TM_QW3_HV_PHYS + TM_NSR];
815 data[0x1] = regs[TM_QW3_HV_PHYS + TM_CPPR];
816 data[0x2] = regs[TM_QW3_HV_PHYS + TM_IPB];
817 data[0x3] = regs[TM_QW2_HV_POOL + TM_IPB];
818 data[0x4] = regs[TM_QW1_OS + TM_ACK_CNT];
819 data[0x5] = regs[TM_QW3_HV_PHYS + TM_LGS];
820 data[0x6] = 0xFF;
821 data[0x7] = regs[TM_QW3_HV_PHYS + TM_WORD2] & 0x80;
822 data[0x7] |= (regs[TM_QW2_HV_POOL + TM_WORD2] & 0x80) >> 1;
823 data[0x7] |= (regs[TM_QW1_OS + TM_WORD2] & 0x80) >> 2;
824 data[0x7] |= (regs[TM_QW3_HV_PHYS + TM_WORD2] & 0x3);
825 data[0x8] = regs[TM_QW1_OS + TM_NSR];
826 data[0x9] = regs[TM_QW1_OS + TM_CPPR];
827 data[0xA] = regs[TM_QW1_OS + TM_IPB];
828 data[0xB] = regs[TM_QW1_OS + TM_LGS];
829 if (regs[TM_QW0_USER + TM_WORD2] & 0x80) {
830 /*
831 * Logical server extension, except VU bit replaced by EB bit
832 * from NSR
833 */
834 data[0xC] = regs[TM_QW0_USER + TM_WORD2];
835 data[0xC] &= ~0x80;
836 data[0xC] |= regs[TM_QW0_USER + TM_NSR] & 0x80;
837 data[0xD] = regs[TM_QW0_USER + TM_WORD2 + 1];
838 data[0xE] = regs[TM_QW0_USER + TM_WORD2 + 2];
839 data[0xF] = regs[TM_QW0_USER + TM_WORD2 + 3];
840 }
841 }
842
843 static void xive2_tm_pull_ctx_ol(XivePresenter *xptr, XiveTCTX *tctx,
844 hwaddr offset, uint64_t value,
845 unsigned size, uint8_t ring)
846 {
847 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
848 uint32_t hw_cam, nvp_idx, xive2_cfg, reserved;
849 uint8_t nvp_blk;
850 Xive2Nvp nvp;
851 uint64_t phys_addr;
852 MemTxResult result;
853
854 hw_cam = xive2_tctx_hw_cam_line(xptr, tctx);
855 nvp_blk = xive2_nvp_blk(hw_cam);
856 nvp_idx = xive2_nvp_idx(hw_cam);
857
858 if (xive2_router_get_nvp(xrtr, nvp_blk, nvp_idx, &nvp)) {
859 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n",
860 nvp_blk, nvp_idx);
861 return;
862 }
863
864 if (!xive2_nvp_is_valid(&nvp)) {
865 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
866 nvp_blk, nvp_idx);
867 return;
868 }
869
870 xive2_cfg = xive2_router_get_config(xrtr);
871
872 phys_addr = xive2_nvp_reporting_addr(&nvp) + 0x80; /* odd line */
873 if (xive2_cfg & XIVE2_GEN1_TIMA_OS) {
874 uint8_t pull_ctxt[REPORT_LINE_GEN1_SIZE];
875
876 xive2_tm_report_line_gen1(tctx, pull_ctxt, REPORT_LINE_GEN1_SIZE);
877 result = dma_memory_write(&address_space_memory, phys_addr,
878 pull_ctxt, REPORT_LINE_GEN1_SIZE,
879 MEMTXATTRS_UNSPECIFIED);
880 assert(result == MEMTX_OK);
881 } else {
882 result = dma_memory_write(&address_space_memory, phys_addr,
883 &tctx->regs, sizeof(tctx->regs),
884 MEMTXATTRS_UNSPECIFIED);
885 assert(result == MEMTX_OK);
886 reserved = 0xFFFFFFFF;
887 result = dma_memory_write(&address_space_memory, phys_addr + 12,
888 &reserved, sizeof(reserved),
889 MEMTXATTRS_UNSPECIFIED);
890 assert(result == MEMTX_OK);
891 }
892
893 /* the rest is similar to pull context to registers */
894 xive2_tm_pull_ctx(xptr, tctx, offset, size, ring);
895 }
896
897 void xive2_tm_pull_os_ctx_ol(XivePresenter *xptr, XiveTCTX *tctx,
898 hwaddr offset, uint64_t value, unsigned size)
899 {
900 xive2_tm_pull_ctx_ol(xptr, tctx, offset, value, size, TM_QW1_OS);
901 }
902
903
904 void xive2_tm_pull_phys_ctx_ol(XivePresenter *xptr, XiveTCTX *tctx,
905 hwaddr offset, uint64_t value, unsigned size)
906 {
907 xive2_tm_pull_ctx_ol(xptr, tctx, offset, value, size, TM_QW3_HV_PHYS);
908 }
909
910 static uint8_t xive2_tctx_restore_ctx(Xive2Router *xrtr, XiveTCTX *tctx,
911 uint8_t ring,
912 uint8_t nvp_blk, uint32_t nvp_idx,
913 Xive2Nvp *nvp)
914 {
915 CPUPPCState *env = &POWERPC_CPU(tctx->cs)->env;
916 uint32_t pir = env->spr_cb[SPR_PIR].default_value;
917 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, ring);
918 uint8_t *regs = &tctx->regs[ring];
919 uint8_t cppr;
920
921 if (!xive2_nvp_is_hw(nvp)) {
922 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: NVP %x/%x is not HW owned\n",
923 nvp_blk, nvp_idx);
924 return 0;
925 }
926
927 cppr = xive_get_field32(NVP2_W2_CPPR, nvp->w2);
928 nvp->w2 = xive_set_field32(NVP2_W2_CPPR, nvp->w2, 0);
929 xive2_router_write_nvp(xrtr, nvp_blk, nvp_idx, nvp, 2);
930
931 sig_regs[TM_CPPR] = cppr;
932 regs[TM_LSMFB] = xive_get_field32(NVP2_W2_LSMFB, nvp->w2);
933 regs[TM_LGS] = xive_get_field32(NVP2_W2_LGS, nvp->w2);
934 regs[TM_T] = xive_get_field32(NVP2_W2_T, nvp->w2);
935
936 nvp->w1 = xive_set_field32(NVP2_W1_CO, nvp->w1, 1);
937 nvp->w1 = xive_set_field32(NVP2_W1_CO_THRID_VALID, nvp->w1, 1);
938 nvp->w1 = xive_set_field32(NVP2_W1_CO_THRID, nvp->w1, pir);
939
940 /*
941 * Checkout privilege: 0:OS, 1:Pool, 2:Hard
942 *
943 * TODO: we don't support hard push/pull
944 */
945 switch (ring) {
946 case TM_QW1_OS:
947 nvp->w1 = xive_set_field32(NVP2_W1_CO_PRIV, nvp->w1, 0);
948 break;
949 case TM_QW2_HV_POOL:
950 nvp->w1 = xive_set_field32(NVP2_W1_CO_PRIV, nvp->w1, 1);
951 break;
952 default:
953 g_assert_not_reached();
954 }
955
956 xive2_router_write_nvp(xrtr, nvp_blk, nvp_idx, nvp, 1);
957
958 /* return restored CPPR to generate a CPU exception if needed */
959 return cppr;
960 }
961
962 /* Restore TIMA VP context from NVP backlog */
963 static void xive2_tctx_restore_nvp(Xive2Router *xrtr, XiveTCTX *tctx,
964 uint8_t ring,
965 uint8_t nvp_blk, uint32_t nvp_idx,
966 bool do_restore)
967 {
968 uint8_t *regs = &tctx->regs[ring];
969 uint8_t ipb;
970 Xive2Nvp nvp;
971
972 /*
973 * Grab the associated thread interrupt context registers in the
974 * associated NVP
975 */
976 if (xive2_router_get_nvp(xrtr, nvp_blk, nvp_idx, &nvp)) {
977 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n",
978 nvp_blk, nvp_idx);
979 return;
980 }
981
982 if (!xive2_nvp_is_valid(&nvp)) {
983 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
984 nvp_blk, nvp_idx);
985 return;
986 }
987
988 /* Automatically restore thread context registers */
989 if (xive2_router_get_config(xrtr) & XIVE2_VP_SAVE_RESTORE && do_restore) {
990 xive2_tctx_restore_ctx(xrtr, tctx, ring, nvp_blk, nvp_idx, &nvp);
991 }
992
993 ipb = xive_get_field32(NVP2_W2_IPB, nvp.w2);
994 if (ipb) {
995 nvp.w2 = xive_set_field32(NVP2_W2_IPB, nvp.w2, 0);
996 xive2_router_write_nvp(xrtr, nvp_blk, nvp_idx, &nvp, 2);
997 }
998 /* IPB bits in the backlog are merged with the TIMA IPB bits */
999 regs[TM_IPB] |= ipb;
1000 }
1001
1002 /*
1003 * Updating the ring CAM line can trigger a resend of interrupt
1004 */
1005 static void xive2_tm_push_ctx(XivePresenter *xptr, XiveTCTX *tctx,
1006 hwaddr offset, uint64_t value, unsigned size,
1007 uint8_t ring)
1008 {
1009 uint32_t cam;
1010 uint32_t w2;
1011 uint64_t dw1;
1012 uint8_t nvp_blk;
1013 uint32_t nvp_idx;
1014 bool v;
1015 bool do_restore;
1016
1017 if (xive_ring_valid(tctx, ring)) {
1018 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Attempt to push VP to enabled"
1019 " ring 0x%02x\n", ring);
1020 return;
1021 }
1022
1023 /* First update the thead context */
1024 switch (size) {
1025 case 1:
1026 tctx->regs[ring + TM_WORD2] = value & 0xff;
1027 cam = xive2_tctx_hw_cam_line(xptr, tctx);
1028 cam |= ((value & 0xc0) << 24); /* V and H bits */
1029 break;
1030 case 4:
1031 cam = value;
1032 w2 = cpu_to_be32(cam);
1033 memcpy(&tctx->regs[ring + TM_WORD2], &w2, 4);
1034 break;
1035 case 8:
1036 cam = value >> 32;
1037 dw1 = cpu_to_be64(value);
1038 memcpy(&tctx->regs[ring + TM_WORD2], &dw1, 8);
1039 break;
1040 default:
1041 g_assert_not_reached();
1042 }
1043
1044 xive2_cam_decode(cam, &nvp_blk, &nvp_idx, &v, &do_restore);
1045
1046 /* Check the interrupt pending bits */
1047 if (v) {
1048 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
1049 uint8_t cur_ring;
1050
1051 xive2_tctx_restore_nvp(xrtr, tctx, ring,
1052 nvp_blk, nvp_idx, do_restore);
1053
1054 for (cur_ring = TM_QW1_OS; cur_ring <= ring;
1055 cur_ring += XIVE_TM_RING_SIZE) {
1056 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, cur_ring);
1057 uint8_t nsr = sig_regs[TM_NSR];
1058
1059 if (!xive_ring_valid(tctx, cur_ring)) {
1060 continue;
1061 }
1062
1063 if (cur_ring == TM_QW2_HV_POOL) {
1064 if (xive_nsr_indicates_exception(cur_ring, nsr)) {
1065 g_assert(xive_nsr_exception_ring(cur_ring, nsr) ==
1066 TM_QW3_HV_PHYS);
1067 xive2_redistribute(xrtr, tctx,
1068 xive_nsr_exception_ring(ring, nsr));
1069 }
1070 xive2_tctx_process_pending(tctx, TM_QW3_HV_PHYS);
1071 break;
1072 }
1073 xive2_tctx_process_pending(tctx, cur_ring);
1074 }
1075 }
1076 }
1077
1078 void xive2_tm_push_os_ctx(XivePresenter *xptr, XiveTCTX *tctx,
1079 hwaddr offset, uint64_t value, unsigned size)
1080 {
1081 xive2_tm_push_ctx(xptr, tctx, offset, value, size, TM_QW1_OS);
1082 }
1083
1084 void xive2_tm_push_pool_ctx(XivePresenter *xptr, XiveTCTX *tctx,
1085 hwaddr offset, uint64_t value, unsigned size)
1086 {
1087 xive2_tm_push_ctx(xptr, tctx, offset, value, size, TM_QW2_HV_POOL);
1088 }
1089
1090 void xive2_tm_push_phys_ctx(XivePresenter *xptr, XiveTCTX *tctx,
1091 hwaddr offset, uint64_t value, unsigned size)
1092 {
1093 xive2_tm_push_ctx(xptr, tctx, offset, value, size, TM_QW3_HV_PHYS);
1094 }
1095
1096 /* returns -1 if ring is invalid, but still populates block and index */
1097 static int xive2_tctx_get_nvp_indexes(XiveTCTX *tctx, uint8_t ring,
1098 uint8_t *nvp_blk, uint32_t *nvp_idx)
1099 {
1100 uint32_t w2;
1101 uint32_t cam = 0;
1102 int rc = 0;
1103
1104 w2 = xive_tctx_word2(&tctx->regs[ring]);
1105 switch (ring) {
1106 case TM_QW1_OS:
1107 if (!(be32_to_cpu(w2) & TM2_QW1W2_VO)) {
1108 rc = -1;
1109 }
1110 cam = xive_get_field32(TM2_QW1W2_OS_CAM, w2);
1111 break;
1112 case TM_QW2_HV_POOL:
1113 if (!(be32_to_cpu(w2) & TM2_QW2W2_VP)) {
1114 rc = -1;
1115 }
1116 cam = xive_get_field32(TM2_QW2W2_POOL_CAM, w2);
1117 break;
1118 case TM_QW3_HV_PHYS:
1119 if (!(be32_to_cpu(w2) & TM2_QW3W2_VT)) {
1120 rc = -1;
1121 }
1122 cam = xive2_tctx_hw_cam_line(tctx->xptr, tctx);
1123 break;
1124 default:
1125 rc = -1;
1126 }
1127 *nvp_blk = xive2_nvp_blk(cam);
1128 *nvp_idx = xive2_nvp_idx(cam);
1129 return rc;
1130 }
1131
1132 static void xive2_tctx_accept_el(XivePresenter *xptr, XiveTCTX *tctx,
1133 uint8_t ring, uint8_t cl_ring)
1134 {
1135 uint64_t rd;
1136 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
1137 uint32_t nvp_idx, xive2_cfg;
1138 uint8_t nvp_blk;
1139 Xive2Nvp nvp;
1140 uint64_t phys_addr;
1141 uint8_t OGen = 0;
1142
1143 xive2_tctx_get_nvp_indexes(tctx, cl_ring, &nvp_blk, &nvp_idx);
1144
1145 if (xive2_router_get_nvp(xrtr, (uint8_t)nvp_blk, nvp_idx, &nvp)) {
1146 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n",
1147 nvp_blk, nvp_idx);
1148 return;
1149 }
1150
1151 if (!xive2_nvp_is_valid(&nvp)) {
1152 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
1153 nvp_blk, nvp_idx);
1154 return;
1155 }
1156
1157
1158 rd = xive_tctx_accept(tctx, ring);
1159
1160 if (ring == TM_QW1_OS) {
1161 OGen = tctx->regs[ring + TM_OGEN];
1162 }
1163 xive2_cfg = xive2_router_get_config(xrtr);
1164 phys_addr = xive2_nvp_reporting_addr(&nvp);
1165 uint8_t report_data[REPORT_LINE_GEN1_SIZE];
1166 memset(report_data, 0xff, sizeof(report_data));
1167 if ((OGen == 1) || (xive2_cfg & XIVE2_GEN1_TIMA_OS)) {
1168 report_data[8] = (rd >> 8) & 0xff;
1169 report_data[9] = rd & 0xff;
1170 } else {
1171 report_data[0] = (rd >> 8) & 0xff;
1172 report_data[1] = rd & 0xff;
1173 }
1174 physical_memory_write(phys_addr, report_data, REPORT_LINE_GEN1_SIZE);
1175 }
1176
1177 void xive2_tm_ack_os_el(XivePresenter *xptr, XiveTCTX *tctx,
1178 hwaddr offset, uint64_t value, unsigned size)
1179 {
1180 xive2_tctx_accept_el(xptr, tctx, TM_QW1_OS, TM_QW1_OS);
1181 }
1182
1183 /* Re-calculate and present pending interrupts */
1184 static void xive2_tctx_process_pending(XiveTCTX *tctx, uint8_t sig_ring)
1185 {
1186 uint8_t *sig_regs = &tctx->regs[sig_ring];
1187 Xive2Router *xrtr = XIVE2_ROUTER(tctx->xptr);
1188 uint8_t backlog_prio;
1189 uint8_t first_group;
1190 uint8_t group_level;
1191 uint8_t pipr_min;
1192 uint8_t lsmfb_min;
1193 uint8_t ring_min;
1194 uint8_t cppr = sig_regs[TM_CPPR];
1195 bool group_enabled;
1196 Xive2Nvp nvp;
1197 int rc;
1198
1199 g_assert(sig_ring == TM_QW3_HV_PHYS || sig_ring == TM_QW1_OS);
1200 g_assert(sig_regs[TM_WORD2] & 0x80);
1201 g_assert(!xive_nsr_indicates_group_exception(sig_ring, sig_regs[TM_NSR]));
1202
1203 /*
1204 * Recompute the PIPR based on local pending interrupts. It will
1205 * be adjusted below if needed in case of pending group interrupts.
1206 */
1207 again:
1208 pipr_min = xive_ipb_to_pipr(sig_regs[TM_IPB]);
1209 group_enabled = !!sig_regs[TM_LGS];
1210 lsmfb_min = group_enabled ? sig_regs[TM_LSMFB] : 0xff;
1211 ring_min = sig_ring;
1212 group_level = 0;
1213
1214 /* PHYS updates also depend on POOL values */
1215 if (sig_ring == TM_QW3_HV_PHYS) {
1216 uint8_t *pool_regs = &tctx->regs[TM_QW2_HV_POOL];
1217
1218 /* POOL values only matter if POOL ctx is valid */
1219 if (pool_regs[TM_WORD2] & 0x80) {
1220 uint8_t pool_pipr = xive_ipb_to_pipr(pool_regs[TM_IPB]);
1221 uint8_t pool_lsmfb = pool_regs[TM_LSMFB];
1222
1223 /*
1224 * Determine highest priority interrupt and
1225 * remember which ring has it.
1226 */
1227 if (pool_pipr < pipr_min) {
1228 pipr_min = pool_pipr;
1229 if (pool_pipr < lsmfb_min) {
1230 ring_min = TM_QW2_HV_POOL;
1231 }
1232 }
1233
1234 /* Values needed for group priority calculation */
1235 if (pool_regs[TM_LGS] && (pool_lsmfb < lsmfb_min)) {
1236 group_enabled = true;
1237 lsmfb_min = pool_lsmfb;
1238 if (lsmfb_min < pipr_min) {
1239 ring_min = TM_QW2_HV_POOL;
1240 }
1241 }
1242 }
1243 }
1244
1245 if (group_enabled &&
1246 lsmfb_min < cppr &&
1247 lsmfb_min < pipr_min) {
1248
1249 uint8_t nvp_blk;
1250 uint32_t nvp_idx;
1251
1252 /*
1253 * Thread has seen a group interrupt with a higher priority
1254 * than the new cppr or pending local interrupt. Check the
1255 * backlog
1256 */
1257 rc = xive2_tctx_get_nvp_indexes(tctx, ring_min, &nvp_blk, &nvp_idx);
1258 if (rc) {
1259 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: set CPPR on invalid "
1260 "context\n");
1261 return;
1262 }
1263
1264 if (xive2_router_get_nvp(xrtr, nvp_blk, nvp_idx, &nvp)) {
1265 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No NVP %x/%x\n",
1266 nvp_blk, nvp_idx);
1267 return;
1268 }
1269
1270 if (!xive2_nvp_is_valid(&nvp)) {
1271 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
1272 nvp_blk, nvp_idx);
1273 return;
1274 }
1275
1276 first_group = xive_get_field32(NVP2_W0_PGOFIRST, nvp.w0);
1277 if (!first_group) {
1278 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid NVP %x/%x\n",
1279 nvp_blk, nvp_idx);
1280 return;
1281 }
1282
1283 backlog_prio = xive2_presenter_backlog_scan(tctx->xptr,
1284 nvp_blk, nvp_idx,
1285 first_group, &group_level);
1286 tctx->regs[ring_min + TM_LSMFB] = backlog_prio;
1287 if (backlog_prio != lsmfb_min) {
1288 /*
1289 * If the group backlog scan finds a less favored or no interrupt,
1290 * then re-do the processing which may turn up a more favored
1291 * interrupt from IPB or the other pool. Backlog should not
1292 * find a priority < LSMFB.
1293 */
1294 g_assert(backlog_prio >= lsmfb_min);
1295 goto again;
1296 }
1297
1298 xive2_presenter_backlog_decr(tctx->xptr, nvp_blk, nvp_idx,
1299 backlog_prio, group_level);
1300 pipr_min = backlog_prio;
1301 }
1302
1303 if (pipr_min > cppr) {
1304 pipr_min = cppr;
1305 }
1306 xive_tctx_pipr_set(tctx, ring_min, pipr_min, group_level);
1307 }
1308
1309 /* NOTE: CPPR only exists for TM_QW1_OS and TM_QW3_HV_PHYS */
1310 static void xive2_tctx_set_cppr(XiveTCTX *tctx, uint8_t sig_ring, uint8_t cppr)
1311 {
1312 uint8_t *sig_regs = &tctx->regs[sig_ring];
1313 Xive2Router *xrtr = XIVE2_ROUTER(tctx->xptr);
1314 uint8_t old_cppr;
1315 uint8_t nsr = sig_regs[TM_NSR];
1316
1317 g_assert(sig_ring == TM_QW1_OS || sig_ring == TM_QW3_HV_PHYS);
1318
1319 g_assert(tctx->regs[TM_QW2_HV_POOL + TM_NSR] == 0);
1320 g_assert(tctx->regs[TM_QW2_HV_POOL + TM_PIPR] == 0);
1321 g_assert(tctx->regs[TM_QW2_HV_POOL + TM_CPPR] == 0);
1322
1323 /* XXX: should show pool IPB for PHYS ring */
1324 trace_xive_tctx_set_cppr(tctx->cs->cpu_index, sig_ring,
1325 sig_regs[TM_IPB], sig_regs[TM_PIPR],
1326 cppr, nsr);
1327
1328 if (cppr > XIVE_PRIORITY_MAX) {
1329 cppr = 0xff;
1330 }
1331
1332 old_cppr = sig_regs[TM_CPPR];
1333 sig_regs[TM_CPPR] = cppr;
1334
1335 /* Handle increased CPPR priority (lower value) */
1336 if (cppr < old_cppr) {
1337 if (cppr <= sig_regs[TM_PIPR]) {
1338 /* CPPR lowered below PIPR, must un-present interrupt */
1339 if (xive_nsr_indicates_exception(sig_ring, nsr)) {
1340 if (xive_nsr_indicates_group_exception(sig_ring, nsr)) {
1341 /* redistribute precluded active grp interrupt */
1342 xive2_redistribute(xrtr, tctx,
1343 xive_nsr_exception_ring(sig_ring, nsr));
1344 return;
1345 }
1346 }
1347
1348 /* interrupt is VP directed, pending in IPB */
1349 xive_tctx_pipr_set(tctx, sig_ring, cppr, 0);
1350 return;
1351 } else {
1352 /* CPPR was lowered, but still above PIPR. No action needed. */
1353 return;
1354 }
1355 }
1356
1357 /* CPPR didn't change, nothing needs to be done */
1358 if (cppr == old_cppr) {
1359 return;
1360 }
1361
1362 /* CPPR priority decreased (higher value) */
1363 if (!xive_nsr_indicates_exception(sig_ring, nsr)) {
1364 xive2_tctx_process_pending(tctx, sig_ring);
1365 }
1366 }
1367
1368 void xive2_tm_set_hv_cppr(XivePresenter *xptr, XiveTCTX *tctx,
1369 hwaddr offset, uint64_t value, unsigned size)
1370 {
1371 xive2_tctx_set_cppr(tctx, TM_QW3_HV_PHYS, value & 0xff);
1372 }
1373
1374 void xive2_tm_set_os_cppr(XivePresenter *xptr, XiveTCTX *tctx,
1375 hwaddr offset, uint64_t value, unsigned size)
1376 {
1377 xive2_tctx_set_cppr(tctx, TM_QW1_OS, value & 0xff);
1378 }
1379
1380 /*
1381 * Adjust the IPB to allow a CPU to process event queues of other
1382 * priorities during one physical interrupt cycle.
1383 */
1384 void xive2_tm_set_os_pending(XivePresenter *xptr, XiveTCTX *tctx,
1385 hwaddr offset, uint64_t value, unsigned size)
1386 {
1387 Xive2Router *xrtr = XIVE2_ROUTER(xptr);
1388 uint8_t ring = TM_QW1_OS;
1389 uint8_t *regs = &tctx->regs[ring];
1390 uint8_t priority = value & 0xff;
1391
1392 /*
1393 * XXX: should this simply set a bit in IPB and wait for it to be picked
1394 * up next cycle, or is it supposed to present it now? We implement the
1395 * latter here.
1396 */
1397 regs[TM_IPB] |= xive_priority_to_ipb(priority);
1398 if (xive_ipb_to_pipr(regs[TM_IPB]) >= regs[TM_PIPR]) {
1399 return;
1400 }
1401 if (xive_nsr_indicates_group_exception(ring, regs[TM_NSR])) {
1402 xive2_redistribute(xrtr, tctx, ring);
1403 }
1404
1405 xive_tctx_pipr_present(tctx, ring, priority, 0);
1406 }
1407
1408 static void xive2_tctx_set_target(XiveTCTX *tctx, uint8_t ring, uint8_t target)
1409 {
1410 uint8_t *regs = &tctx->regs[ring];
1411
1412 regs[TM_T] = target;
1413 }
1414
1415 void xive2_tm_set_hv_target(XivePresenter *xptr, XiveTCTX *tctx,
1416 hwaddr offset, uint64_t value, unsigned size)
1417 {
1418 xive2_tctx_set_target(tctx, TM_QW3_HV_PHYS, value & 0xff);
1419 }
1420
1421 /*
1422 * XIVE Router (aka. Virtualization Controller or IVRE)
1423 */
1424
1425 int xive2_router_get_eas(Xive2Router *xrtr, uint8_t eas_blk, uint32_t eas_idx,
1426 Xive2Eas *eas)
1427 {
1428 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1429
1430 return xrc->get_eas(xrtr, eas_blk, eas_idx, eas);
1431 }
1432
1433 static
1434 int xive2_router_get_pq(Xive2Router *xrtr, uint8_t eas_blk, uint32_t eas_idx,
1435 uint8_t *pq)
1436 {
1437 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1438
1439 return xrc->get_pq(xrtr, eas_blk, eas_idx, pq);
1440 }
1441
1442 static
1443 int xive2_router_set_pq(Xive2Router *xrtr, uint8_t eas_blk, uint32_t eas_idx,
1444 uint8_t *pq)
1445 {
1446 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1447
1448 return xrc->set_pq(xrtr, eas_blk, eas_idx, pq);
1449 }
1450
1451 int xive2_router_get_end(Xive2Router *xrtr, uint8_t end_blk, uint32_t end_idx,
1452 Xive2End *end)
1453 {
1454 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1455
1456 return xrc->get_end(xrtr, end_blk, end_idx, end);
1457 }
1458
1459 int xive2_router_write_end(Xive2Router *xrtr, uint8_t end_blk, uint32_t end_idx,
1460 Xive2End *end, uint8_t word_number)
1461 {
1462 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1463
1464 return xrc->write_end(xrtr, end_blk, end_idx, end, word_number);
1465 }
1466
1467 int xive2_router_get_nvp(Xive2Router *xrtr, uint8_t nvp_blk, uint32_t nvp_idx,
1468 Xive2Nvp *nvp)
1469 {
1470 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1471
1472 return xrc->get_nvp(xrtr, nvp_blk, nvp_idx, nvp);
1473 }
1474
1475 int xive2_router_write_nvp(Xive2Router *xrtr, uint8_t nvp_blk, uint32_t nvp_idx,
1476 Xive2Nvp *nvp, uint8_t word_number)
1477 {
1478 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1479
1480 return xrc->write_nvp(xrtr, nvp_blk, nvp_idx, nvp, word_number);
1481 }
1482
1483 int xive2_router_get_nvgc(Xive2Router *xrtr, bool crowd,
1484 uint8_t nvgc_blk, uint32_t nvgc_idx,
1485 Xive2Nvgc *nvgc)
1486 {
1487 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1488
1489 return xrc->get_nvgc(xrtr, crowd, nvgc_blk, nvgc_idx, nvgc);
1490 }
1491
1492 int xive2_router_write_nvgc(Xive2Router *xrtr, bool crowd,
1493 uint8_t nvgc_blk, uint32_t nvgc_idx,
1494 Xive2Nvgc *nvgc)
1495 {
1496 Xive2RouterClass *xrc = XIVE2_ROUTER_GET_CLASS(xrtr);
1497
1498 return xrc->write_nvgc(xrtr, crowd, nvgc_blk, nvgc_idx, nvgc);
1499 }
1500
1501 static bool xive2_vp_match_mask(uint32_t cam1, uint32_t cam2,
1502 uint32_t vp_mask)
1503 {
1504 return (cam1 & vp_mask) == (cam2 & vp_mask);
1505 }
1506
1507 static uint8_t xive2_get_vp_block_mask(uint32_t nvt_blk, bool crowd)
1508 {
1509 uint8_t block_mask = 0b1111;
1510
1511 /* 3 supported crowd sizes: 2, 4, 16 */
1512 if (crowd) {
1513 uint32_t size = xive_get_vpgroup_size(nvt_blk);
1514
1515 if (size != 2 && size != 4 && size != 16) {
1516 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid crowd size of %d",
1517 size);
1518 return block_mask;
1519 }
1520 block_mask &= ~(size - 1);
1521 }
1522 return block_mask;
1523 }
1524
1525 static uint32_t xive2_get_vp_index_mask(uint32_t nvt_index, bool cam_ignore)
1526 {
1527 uint32_t index_mask = 0xFFFFFF; /* 24 bits */
1528
1529 if (cam_ignore) {
1530 uint32_t size = xive_get_vpgroup_size(nvt_index);
1531
1532 if (size < 2) {
1533 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid group size of %d",
1534 size);
1535 return index_mask;
1536 }
1537 index_mask &= ~(size - 1);
1538 }
1539 return index_mask;
1540 }
1541
1542 /*
1543 * The thread context register words are in big-endian format.
1544 */
1545 int xive2_presenter_tctx_match(XivePresenter *xptr, XiveTCTX *tctx,
1546 uint8_t format,
1547 uint8_t nvt_blk, uint32_t nvt_idx,
1548 bool crowd, bool cam_ignore,
1549 uint32_t logic_serv)
1550 {
1551 uint32_t cam = xive2_nvp_cam_line(nvt_blk, nvt_idx);
1552 uint32_t qw3w2 = xive_tctx_word2(&tctx->regs[TM_QW3_HV_PHYS]);
1553 uint32_t qw2w2 = xive_tctx_word2(&tctx->regs[TM_QW2_HV_POOL]);
1554 uint32_t qw1w2 = xive_tctx_word2(&tctx->regs[TM_QW1_OS]);
1555 uint32_t qw0w2 = xive_tctx_word2(&tctx->regs[TM_QW0_USER]);
1556
1557 uint32_t index_mask, vp_mask;
1558 uint8_t block_mask;
1559
1560 if (format == 0) {
1561 /*
1562 * i=0: Specific NVT notification
1563 * i=1: VP-group notification (bits ignored at the end of the
1564 * NVT identifier)
1565 */
1566 block_mask = xive2_get_vp_block_mask(nvt_blk, crowd);
1567 index_mask = xive2_get_vp_index_mask(nvt_idx, cam_ignore);
1568 vp_mask = xive2_nvp_cam_line(block_mask, index_mask);
1569
1570 /* For VP-group notifications, threads with LGS=0 are excluded */
1571
1572 /* PHYS ring */
1573 if ((be32_to_cpu(qw3w2) & TM2_QW3W2_VT) &&
1574 !(cam_ignore && tctx->regs[TM_QW3_HV_PHYS + TM_LGS] == 0) &&
1575 xive2_vp_match_mask(cam,
1576 xive2_tctx_hw_cam_line(xptr, tctx),
1577 vp_mask)) {
1578 return TM_QW3_HV_PHYS;
1579 }
1580
1581 /* HV POOL ring */
1582 if ((be32_to_cpu(qw2w2) & TM2_QW2W2_VP) &&
1583 !(cam_ignore && tctx->regs[TM_QW2_HV_POOL + TM_LGS] == 0) &&
1584 xive2_vp_match_mask(cam,
1585 xive_get_field32(TM2_QW2W2_POOL_CAM, qw2w2),
1586 vp_mask)) {
1587 return TM_QW2_HV_POOL;
1588 }
1589
1590 /* OS ring */
1591 if ((be32_to_cpu(qw1w2) & TM2_QW1W2_VO) &&
1592 !(cam_ignore && tctx->regs[TM_QW1_OS + TM_LGS] == 0) &&
1593 xive2_vp_match_mask(cam,
1594 xive_get_field32(TM2_QW1W2_OS_CAM, qw1w2),
1595 vp_mask)) {
1596 return TM_QW1_OS;
1597 }
1598 } else {
1599 /* F=1 : User level Event-Based Branch (EBB) notification */
1600
1601 /* FIXME: what if cam_ignore and LGS = 0 ? */
1602 /* USER ring */
1603 if ((be32_to_cpu(qw1w2) & TM2_QW1W2_VO) &&
1604 (cam == xive_get_field32(TM2_QW1W2_OS_CAM, qw1w2)) &&
1605 (be32_to_cpu(qw0w2) & TM2_QW0W2_VU) &&
1606 (logic_serv == xive_get_field32(TM2_QW0W2_LOGIC_SERV, qw0w2))) {
1607 return TM_QW0_USER;
1608 }
1609 }
1610 return -1;
1611 }
1612
1613 bool xive2_tm_irq_precluded(XiveTCTX *tctx, int ring, uint8_t priority)
1614 {
1615 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, ring);
1616
1617 /*
1618 * The xive2_presenter_tctx_match() above tells if there's a match
1619 * but for VP-group notification, we still need to look at the
1620 * priority to know if the thread can take the interrupt now or if
1621 * it is precluded.
1622 */
1623 if (priority < sig_regs[TM_PIPR]) {
1624 return false;
1625 }
1626 return true;
1627 }
1628
1629 void xive2_tm_set_lsmfb(XiveTCTX *tctx, int ring, uint8_t priority)
1630 {
1631 uint8_t *regs = &tctx->regs[ring];
1632
1633 /*
1634 * Called by the router during a VP-group notification when the
1635 * thread matches but can't take the interrupt because it's
1636 * already running at a more favored priority. It then stores the
1637 * new interrupt priority in the LSMFB field.
1638 */
1639 regs[TM_LSMFB] = priority;
1640 }
1641
1642 static void xive2_router_realize(DeviceState *dev, Error **errp)
1643 {
1644 Xive2Router *xrtr = XIVE2_ROUTER(dev);
1645
1646 assert(xrtr->xfb);
1647 }
1648
1649 /*
1650 * Notification using the END ESe/ESn bit (Event State Buffer for
1651 * escalation and notification). Profide further coalescing in the
1652 * Router.
1653 */
1654 static bool xive2_router_end_es_notify(Xive2Router *xrtr, uint8_t end_blk,
1655 uint32_t end_idx, Xive2End *end,
1656 uint32_t end_esmask)
1657 {
1658 uint8_t pq = xive_get_field32(end_esmask, end->w1);
1659 bool notify = xive_esb_trigger(&pq);
1660
1661 if (pq != xive_get_field32(end_esmask, end->w1)) {
1662 end->w1 = xive_set_field32(end_esmask, end->w1, pq);
1663 xive2_router_write_end(xrtr, end_blk, end_idx, end, 1);
1664 }
1665
1666 /* ESe/n[Q]=1 : end of notification */
1667 return notify;
1668 }
1669
1670 /*
1671 * An END trigger can come from an event trigger (IPI or HW) or from
1672 * another chip. We don't model the PowerBus but the END trigger
1673 * message has the same parameters than in the function below.
1674 */
1675 static void xive2_router_end_notify(Xive2Router *xrtr, uint8_t end_blk,
1676 uint32_t end_idx, uint32_t end_data,
1677 bool redistribute)
1678 {
1679 Xive2End end;
1680 uint8_t priority;
1681 uint8_t format;
1682 XiveTCTXMatch match;
1683 bool crowd, cam_ignore;
1684 uint8_t nvx_blk;
1685 uint32_t nvx_idx;
1686
1687 /* END cache lookup */
1688 if (xive2_router_get_end(xrtr, end_blk, end_idx, &end)) {
1689 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No END %x/%x\n", end_blk,
1690 end_idx);
1691 return;
1692 }
1693
1694 if (!xive2_end_is_valid(&end)) {
1695 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: END %x/%x is invalid\n",
1696 end_blk, end_idx);
1697 return;
1698 }
1699
1700 if (xive2_end_is_crowd(&end) && !xive2_end_is_ignore(&end)) {
1701 qemu_log_mask(LOG_GUEST_ERROR,
1702 "XIVE: invalid END, 'crowd' bit requires 'ignore' bit\n");
1703 return;
1704 }
1705
1706 if (!redistribute && xive2_end_is_enqueue(&end)) {
1707 trace_xive_end_enqueue(end_blk, end_idx, end_data);
1708 xive2_end_enqueue(&end, end_data);
1709 /* Enqueuing event data modifies the EQ toggle and index */
1710 xive2_router_write_end(xrtr, end_blk, end_idx, &end, 1);
1711 }
1712
1713 /*
1714 * When the END is silent, we skip the notification part.
1715 */
1716 if (xive2_end_is_silent_escalation(&end)) {
1717 goto do_escalation;
1718 }
1719
1720 /*
1721 * The W7 format depends on the F bit in W6. It defines the type
1722 * of the notification :
1723 *
1724 * F=0 : single or multiple NVP notification
1725 * F=1 : User level Event-Based Branch (EBB) notification, no
1726 * priority
1727 */
1728 format = xive_get_field32(END2_W6_FORMAT_BIT, end.w6);
1729 priority = xive_get_field32(END2_W7_F0_PRIORITY, end.w7);
1730
1731 /* The END is masked */
1732 if (format == 0 && priority == 0xff) {
1733 return;
1734 }
1735
1736 /*
1737 * Check the END ESn (Event State Buffer for notification) for
1738 * even further coalescing in the Router
1739 */
1740 if (!xive2_end_is_notify(&end)) {
1741 /* ESn[Q]=1 : end of notification */
1742 if (!xive2_router_end_es_notify(xrtr, end_blk, end_idx,
1743 &end, END2_W1_ESn)) {
1744 return;
1745 }
1746 }
1747
1748 /*
1749 * Follows IVPE notification
1750 */
1751 nvx_blk = xive_get_field32(END2_W6_VP_BLOCK, end.w6);
1752 nvx_idx = xive_get_field32(END2_W6_VP_OFFSET, end.w6);
1753 crowd = xive2_end_is_crowd(&end);
1754 cam_ignore = xive2_end_is_ignore(&end);
1755
1756 /* TODO: Auto EOI. */
1757 if (xive_presenter_match(xrtr->xfb, format, nvx_blk, nvx_idx,
1758 crowd, cam_ignore, priority,
1759 xive_get_field32(END2_W7_F1_LOG_SERVER_ID, end.w7),
1760 &match)) {
1761 XiveTCTX *tctx = match.tctx;
1762 uint8_t ring = match.ring;
1763 uint8_t *sig_regs = xive_tctx_signal_regs(tctx, ring);
1764 uint8_t nsr = sig_regs[TM_NSR];
1765 uint8_t group_level;
1766
1767 if (priority < sig_regs[TM_PIPR] &&
1768 xive_nsr_indicates_group_exception(ring, nsr)) {
1769 xive2_redistribute(xrtr, tctx, xive_nsr_exception_ring(ring, nsr));
1770 }
1771
1772 group_level = xive_get_group_level(crowd, cam_ignore, nvx_blk, nvx_idx);
1773 trace_xive_presenter_notify(nvx_blk, nvx_idx, ring, group_level);
1774 xive_tctx_pipr_present(tctx, ring, priority, group_level);
1775 return;
1776 }
1777
1778 /*
1779 * If no matching NVP is dispatched on a HW thread :
1780 * - specific VP: update the NVP structure if backlog is activated
1781 * - VP-group: update the backlog counter for that priority in the NVG
1782 */
1783 if (xive2_end_is_backlog(&end)) {
1784
1785 if (format == 1) {
1786 qemu_log_mask(LOG_GUEST_ERROR,
1787 "XIVE: END %x/%x invalid config: F1 & backlog\n",
1788 end_blk, end_idx);
1789 return;
1790 }
1791
1792 if (!cam_ignore) {
1793 uint8_t ipb;
1794 Xive2Nvp nvp;
1795
1796 /* NVP cache lookup */
1797 if (xive2_router_get_nvp(xrtr, nvx_blk, nvx_idx, &nvp)) {
1798 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: no NVP %x/%x\n",
1799 nvx_blk, nvx_idx);
1800 return;
1801 }
1802
1803 if (!xive2_nvp_is_valid(&nvp)) {
1804 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: NVP %x/%x is invalid\n",
1805 nvx_blk, nvx_idx);
1806 return;
1807 }
1808
1809 /*
1810 * Record the IPB in the associated NVP structure for later
1811 * use. The presenter will resend the interrupt when the vCPU
1812 * is dispatched again on a HW thread.
1813 */
1814 ipb = xive_get_field32(NVP2_W2_IPB, nvp.w2) |
1815 xive_priority_to_ipb(priority);
1816 nvp.w2 = xive_set_field32(NVP2_W2_IPB, nvp.w2, ipb);
1817 xive2_router_write_nvp(xrtr, nvx_blk, nvx_idx, &nvp, 2);
1818 } else {
1819 Xive2Nvgc nvgc;
1820 uint32_t backlog;
1821
1822 /*
1823 * For groups and crowds, the per-priority backlog
1824 * counters are stored in the NVG/NVC structures
1825 */
1826 if (xive2_router_get_nvgc(xrtr, crowd,
1827 nvx_blk, nvx_idx, &nvgc)) {
1828 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: no %s %x/%x\n",
1829 crowd ? "NVC" : "NVG", nvx_blk, nvx_idx);
1830 return;
1831 }
1832
1833 if (!xive2_nvgc_is_valid(&nvgc)) {
1834 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: NVG %x/%x is invalid\n",
1835 nvx_blk, nvx_idx);
1836 return;
1837 }
1838
1839 /*
1840 * Increment the backlog counter for that priority.
1841 * We only call broadcast the first time the counter is
1842 * incremented. broadcast will set the LSMFB field of the TIMA of
1843 * relevant threads so that they know an interrupt is pending.
1844 */
1845 backlog = xive2_nvgc_get_backlog(&nvgc, priority) + 1;
1846 xive2_nvgc_set_backlog(&nvgc, priority, backlog);
1847 xive2_router_write_nvgc(xrtr, crowd, nvx_blk, nvx_idx, &nvgc);
1848
1849 if (backlog == 1) {
1850 XiveFabricClass *xfc = XIVE_FABRIC_GET_CLASS(xrtr->xfb);
1851 xfc->broadcast(xrtr->xfb, nvx_blk, nvx_idx,
1852 crowd, cam_ignore, priority);
1853
1854 if (!xive2_end_is_precluded_escalation(&end)) {
1855 /*
1856 * The interrupt will be picked up when the
1857 * matching thread lowers its priority level
1858 */
1859 return;
1860 }
1861 }
1862 }
1863 }
1864
1865 do_escalation:
1866 /*
1867 * If activated, escalate notification using the ESe PQ bits and
1868 * the EAS in w4-5
1869 */
1870 if (!xive2_end_is_escalate(&end)) {
1871 return;
1872 }
1873
1874 /*
1875 * Check the END ESe (Event State Buffer for escalation) for even
1876 * further coalescing in the Router
1877 */
1878 if (!xive2_end_is_uncond_escalation(&end)) {
1879 /* ESe[Q]=1 : end of escalation notification */
1880 if (!xive2_router_end_es_notify(xrtr, end_blk, end_idx,
1881 &end, END2_W1_ESe)) {
1882 return;
1883 }
1884 }
1885
1886 if (xive2_end_is_escalate_end(&end)) {
1887 /*
1888 * Perform END Adaptive escalation processing
1889 * The END trigger becomes an Escalation trigger
1890 */
1891 uint8_t esc_blk = xive_get_field32(END2_W4_END_BLOCK, end.w4);
1892 uint32_t esc_idx = xive_get_field32(END2_W4_ESC_END_INDEX, end.w4);
1893 uint32_t esc_data = xive_get_field32(END2_W5_ESC_END_DATA, end.w5);
1894 trace_xive_escalate_end(end_blk, end_idx, esc_blk, esc_idx, esc_data);
1895 xive2_router_end_notify(xrtr, esc_blk, esc_idx, esc_data, false);
1896 } /* end END adaptive escalation */
1897
1898 else {
1899 uint32_t lisn; /* Logical Interrupt Source Number */
1900
1901 /*
1902 * Perform ESB escalation processing
1903 * E[N] == 1 --> N
1904 * Req[Block] <- E[ESB_Block]
1905 * Req[Index] <- E[ESB_Index]
1906 * Req[Offset] <- 0x000
1907 * Execute <ESB Store> Req command
1908 */
1909 lisn = XIVE_EAS(xive_get_field32(END2_W4_END_BLOCK, end.w4),
1910 xive_get_field32(END2_W4_ESC_END_INDEX, end.w4));
1911
1912 trace_xive_escalate_esb(end_blk, end_idx, lisn);
1913 xive2_notify(xrtr, lisn, true /* pq_checked */);
1914 }
1915
1916 return;
1917 }
1918
1919 void xive2_notify(Xive2Router *xrtr , uint32_t lisn, bool pq_checked)
1920 {
1921 uint8_t eas_blk = XIVE_EAS_BLOCK(lisn);
1922 uint32_t eas_idx = XIVE_EAS_INDEX(lisn);
1923 Xive2Eas eas;
1924
1925 /* EAS cache lookup */
1926 if (xive2_router_get_eas(xrtr, eas_blk, eas_idx, &eas)) {
1927 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Unknown LISN %x\n", lisn);
1928 return;
1929 }
1930
1931 if (!pq_checked) {
1932 bool notify;
1933 uint8_t pq;
1934
1935 /* PQ cache lookup */
1936 if (xive2_router_get_pq(xrtr, eas_blk, eas_idx, &pq)) {
1937 /* Set FIR */
1938 g_assert_not_reached();
1939 }
1940
1941 notify = xive_esb_trigger(&pq);
1942
1943 if (xive2_router_set_pq(xrtr, eas_blk, eas_idx, &pq)) {
1944 /* Set FIR */
1945 g_assert_not_reached();
1946 }
1947
1948 if (!notify) {
1949 return;
1950 }
1951 }
1952
1953 if (!xive2_eas_is_valid(&eas)) {
1954 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: Invalid LISN %x\n", lisn);
1955 return;
1956 }
1957
1958 if (xive2_eas_is_masked(&eas)) {
1959 /* Notification completed */
1960 return;
1961 }
1962
1963 /* TODO: add support for EAS resume */
1964 if (xive2_eas_is_resume(&eas)) {
1965 qemu_log_mask(LOG_UNIMP,
1966 "XIVE: EAS resume processing unimplemented - LISN %x\n",
1967 lisn);
1968 return;
1969 }
1970
1971 /*
1972 * The event trigger becomes an END trigger
1973 */
1974 xive2_router_end_notify(xrtr,
1975 xive_get_field64(EAS2_END_BLOCK, eas.w),
1976 xive_get_field64(EAS2_END_INDEX, eas.w),
1977 xive_get_field64(EAS2_END_DATA, eas.w),
1978 false);
1979 return;
1980 }
1981
1982 void xive2_router_notify(XiveNotifier *xn, uint32_t lisn, bool pq_checked)
1983 {
1984 Xive2Router *xrtr = XIVE2_ROUTER(xn);
1985
1986 xive2_notify(xrtr, lisn, pq_checked);
1987 return;
1988 }
1989
1990 static const Property xive2_router_properties[] = {
1991 DEFINE_PROP_LINK("xive-fabric", Xive2Router, xfb,
1992 TYPE_XIVE_FABRIC, XiveFabric *),
1993 };
1994
1995 static void xive2_router_class_init(ObjectClass *klass, const void *data)
1996 {
1997 DeviceClass *dc = DEVICE_CLASS(klass);
1998 XiveNotifierClass *xnc = XIVE_NOTIFIER_CLASS(klass);
1999
2000 dc->desc = "XIVE2 Router Engine";
2001 device_class_set_props(dc, xive2_router_properties);
2002 /* Parent is SysBusDeviceClass. No need to call its realize hook */
2003 dc->realize = xive2_router_realize;
2004 xnc->notify = xive2_router_notify;
2005 }
2006
2007 static const TypeInfo xive2_router_info = {
2008 .name = TYPE_XIVE2_ROUTER,
2009 .parent = TYPE_SYS_BUS_DEVICE,
2010 .abstract = true,
2011 .instance_size = sizeof(Xive2Router),
2012 .class_size = sizeof(Xive2RouterClass),
2013 .class_init = xive2_router_class_init,
2014 .interfaces = (const InterfaceInfo[]) {
2015 { TYPE_XIVE_NOTIFIER },
2016 { TYPE_XIVE_PRESENTER },
2017 { }
2018 }
2019 };
2020
2021 static inline bool addr_is_even(hwaddr addr, uint32_t shift)
2022 {
2023 return !((addr >> shift) & 1);
2024 }
2025
2026 static uint64_t xive2_end_source_read(void *opaque, hwaddr addr, unsigned size)
2027 {
2028 Xive2EndSource *xsrc = XIVE2_END_SOURCE(opaque);
2029 uint32_t offset = addr & 0xFFF;
2030 uint8_t end_blk;
2031 uint32_t end_idx;
2032 Xive2End end;
2033 uint32_t end_esmask;
2034 uint8_t pq;
2035 uint64_t ret;
2036
2037 /*
2038 * The block id should be deduced from the load address on the END
2039 * ESB MMIO but our model only supports a single block per XIVE chip.
2040 */
2041 end_blk = xive2_router_get_block_id(xsrc->xrtr);
2042 end_idx = addr >> (xsrc->esb_shift + 1);
2043
2044 if (xive2_router_get_end(xsrc->xrtr, end_blk, end_idx, &end)) {
2045 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No END %x/%x\n", end_blk,
2046 end_idx);
2047 return -1;
2048 }
2049
2050 if (!xive2_end_is_valid(&end)) {
2051 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: END %x/%x is invalid\n",
2052 end_blk, end_idx);
2053 return -1;
2054 }
2055
2056 end_esmask = addr_is_even(addr, xsrc->esb_shift) ? END2_W1_ESn :
2057 END2_W1_ESe;
2058 pq = xive_get_field32(end_esmask, end.w1);
2059
2060 switch (offset) {
2061 case XIVE_ESB_LOAD_EOI ... XIVE_ESB_LOAD_EOI + 0x7FF:
2062 ret = xive_esb_eoi(&pq);
2063
2064 /* Forward the source event notification for routing ?? */
2065 break;
2066
2067 case XIVE_ESB_GET ... XIVE_ESB_GET + 0x3FF:
2068 ret = pq;
2069 break;
2070
2071 case XIVE_ESB_SET_PQ_00 ... XIVE_ESB_SET_PQ_00 + 0x0FF:
2072 case XIVE_ESB_SET_PQ_01 ... XIVE_ESB_SET_PQ_01 + 0x0FF:
2073 case XIVE_ESB_SET_PQ_10 ... XIVE_ESB_SET_PQ_10 + 0x0FF:
2074 case XIVE_ESB_SET_PQ_11 ... XIVE_ESB_SET_PQ_11 + 0x0FF:
2075 ret = xive_esb_set(&pq, (offset >> 8) & 0x3);
2076 break;
2077 default:
2078 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid END ESB load addr %d\n",
2079 offset);
2080 return -1;
2081 }
2082
2083 if (pq != xive_get_field32(end_esmask, end.w1)) {
2084 end.w1 = xive_set_field32(end_esmask, end.w1, pq);
2085 xive2_router_write_end(xsrc->xrtr, end_blk, end_idx, &end, 1);
2086 }
2087
2088 return ret;
2089 }
2090
2091 static void xive2_end_source_write(void *opaque, hwaddr addr,
2092 uint64_t value, unsigned size)
2093 {
2094 Xive2EndSource *xsrc = XIVE2_END_SOURCE(opaque);
2095 uint32_t offset = addr & 0xFFF;
2096 uint8_t end_blk;
2097 uint32_t end_idx;
2098 Xive2End end;
2099 uint32_t end_esmask;
2100 uint8_t pq;
2101 bool notify = false;
2102
2103 /*
2104 * The block id should be deduced from the load address on the END
2105 * ESB MMIO but our model only supports a single block per XIVE chip.
2106 */
2107 end_blk = xive2_router_get_block_id(xsrc->xrtr);
2108 end_idx = addr >> (xsrc->esb_shift + 1);
2109
2110 if (xive2_router_get_end(xsrc->xrtr, end_blk, end_idx, &end)) {
2111 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: No END %x/%x\n", end_blk,
2112 end_idx);
2113 return;
2114 }
2115
2116 if (!xive2_end_is_valid(&end)) {
2117 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: END %x/%x is invalid\n",
2118 end_blk, end_idx);
2119 return;
2120 }
2121
2122 end_esmask = addr_is_even(addr, xsrc->esb_shift) ? END2_W1_ESn :
2123 END2_W1_ESe;
2124 pq = xive_get_field32(end_esmask, end.w1);
2125
2126 switch (offset) {
2127 case 0 ... 0x3FF:
2128 notify = xive_esb_trigger(&pq);
2129 break;
2130
2131 case XIVE_ESB_STORE_EOI ... XIVE_ESB_STORE_EOI + 0x3FF:
2132 /* TODO: can we check StoreEOI availability from the router ? */
2133 notify = xive_esb_eoi(&pq);
2134 break;
2135
2136 case XIVE_ESB_INJECT ... XIVE_ESB_INJECT + 0x3FF:
2137 if (end_esmask == END2_W1_ESe) {
2138 qemu_log_mask(LOG_GUEST_ERROR,
2139 "XIVE: END %x/%x can not EQ inject on ESe\n",
2140 end_blk, end_idx);
2141 return;
2142 }
2143 notify = true;
2144 break;
2145
2146 default:
2147 qemu_log_mask(LOG_GUEST_ERROR, "XIVE: invalid END ESB write addr %d\n",
2148 offset);
2149 return;
2150 }
2151
2152 if (pq != xive_get_field32(end_esmask, end.w1)) {
2153 end.w1 = xive_set_field32(end_esmask, end.w1, pq);
2154 xive2_router_write_end(xsrc->xrtr, end_blk, end_idx, &end, 1);
2155 }
2156
2157 /* TODO: Forward the source event notification for routing */
2158 if (notify) {
2159 ;
2160 }
2161 }
2162
2163 static const MemoryRegionOps xive2_end_source_ops = {
2164 .read = xive2_end_source_read,
2165 .write = xive2_end_source_write,
2166 .endianness = DEVICE_BIG_ENDIAN,
2167 .valid = {
2168 .min_access_size = 1,
2169 .max_access_size = 8,
2170 },
2171 .impl = {
2172 .min_access_size = 1,
2173 .max_access_size = 8,
2174 },
2175 };
2176
2177 static void xive2_end_source_realize(DeviceState *dev, Error **errp)
2178 {
2179 Xive2EndSource *xsrc = XIVE2_END_SOURCE(dev);
2180
2181 assert(xsrc->xrtr);
2182
2183 if (!xsrc->nr_ends) {
2184 error_setg(errp, "Number of interrupt needs to be greater than 0");
2185 return;
2186 }
2187
2188 if (xsrc->esb_shift != XIVE_ESB_4K &&
2189 xsrc->esb_shift != XIVE_ESB_64K) {
2190 error_setg(errp, "Invalid ESB shift setting");
2191 return;
2192 }
2193
2194 /*
2195 * Each END is assigned an even/odd pair of MMIO pages, the even page
2196 * manages the ESn field while the odd page manages the ESe field.
2197 */
2198 memory_region_init_io(&xsrc->esb_mmio, OBJECT(xsrc),
2199 &xive2_end_source_ops, xsrc, "xive.end",
2200 (1ull << (xsrc->esb_shift + 1)) * xsrc->nr_ends);
2201 }
2202
2203 static const Property xive2_end_source_properties[] = {
2204 DEFINE_PROP_UINT32("nr-ends", Xive2EndSource, nr_ends, 0),
2205 DEFINE_PROP_UINT32("shift", Xive2EndSource, esb_shift, XIVE_ESB_64K),
2206 DEFINE_PROP_LINK("xive", Xive2EndSource, xrtr, TYPE_XIVE2_ROUTER,
2207 Xive2Router *),
2208 };
2209
2210 static void xive2_end_source_class_init(ObjectClass *klass, const void *data)
2211 {
2212 DeviceClass *dc = DEVICE_CLASS(klass);
2213
2214 dc->desc = "XIVE END Source";
2215 device_class_set_props(dc, xive2_end_source_properties);
2216 dc->realize = xive2_end_source_realize;
2217 dc->user_creatable = false;
2218 }
2219
2220 static const TypeInfo xive2_end_source_info = {
2221 .name = TYPE_XIVE2_END_SOURCE,
2222 .parent = TYPE_DEVICE,
2223 .instance_size = sizeof(Xive2EndSource),
2224 .class_init = xive2_end_source_class_init,
2225 };
2226
2227 static void xive2_register_types(void)
2228 {
2229 type_register_static(&xive2_router_info);
2230 type_register_static(&xive2_end_source_info);
2231 }
2232
2233 type_init(xive2_register_types)