master
c 1,155 lines 31.4 KB
Raw
1 /*
2 * Inter-VM Shared Memory PCI device.
3 *
4 * Author:
5 * Cam Macdonell <cam@cs.ualberta.ca>
6 *
7 * Based On: cirrus_vga.c
8 * Copyright (c) 2004 Fabrice Bellard
9 * Copyright (c) 2004 Makoto Suzuki (suzu)
10 *
11 * and rtl8139.c
12 * Copyright (c) 2006 Igor Kovalenko
13 *
14 * This code is licensed under the GNU GPL v2.
15 *
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "qapi/error.h"
23 #include "qemu/cutils.h"
24 #include "hw/pci/pci.h"
25 #include "hw/core/qdev-properties.h"
26 #include "hw/core/qdev-properties-system.h"
27 #include "hw/pci/msi.h"
28 #include "hw/pci/msix.h"
29 #include "system/accel-irq.h"
30 #include "migration/blocker.h"
31 #include "migration/vmstate.h"
32 #include "qemu/error-report.h"
33 #include "qemu/event_notifier.h"
34 #include "qemu/module.h"
35 #include "qom/object_interfaces.h"
36 #include "chardev/char-fe.h"
37 #include "system/hostmem.h"
38 #include "qapi/visitor.h"
39
40 #include "hw/misc/ivshmem.h"
41 #include "qom/object.h"
42
43 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET
44 #define PCI_DEVICE_ID_IVSHMEM 0x1110
45
46 #define IVSHMEM_MAX_PEERS UINT16_MAX
47 #define IVSHMEM_IOEVENTFD 0
48 #define IVSHMEM_MSI 1
49
50 #define IVSHMEM_REG_BAR_SIZE 0x100
51
52 #define IVSHMEM_DEBUG 0
53 #define IVSHMEM_DPRINTF(fmt, ...) \
54 do { \
55 if (IVSHMEM_DEBUG) { \
56 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \
57 } \
58 } while (0)
59
60 #define TYPE_IVSHMEM_COMMON "ivshmem-common"
61 typedef struct IVShmemState IVShmemState;
62 DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_COMMON,
63 TYPE_IVSHMEM_COMMON)
64
65 #define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
66 DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_PLAIN,
67 TYPE_IVSHMEM_PLAIN)
68
69 #define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
70 DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_DOORBELL,
71 TYPE_IVSHMEM_DOORBELL)
72
73 #define TYPE_IVSHMEM "ivshmem"
74 DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM,
75 TYPE_IVSHMEM)
76
77 typedef struct Peer {
78 int nb_eventfds;
79 EventNotifier *eventfds;
80 } Peer;
81
82 typedef struct MSIVector {
83 PCIDevice *pdev;
84 int virq;
85 bool unmasked;
86 } MSIVector;
87
88 struct IVShmemState {
89 /*< private >*/
90 PCIDevice parent_obj;
91 /*< public >*/
92
93 uint32_t features;
94
95 /* exactly one of these two may be set */
96 HostMemoryBackend *hostmem; /* with interrupts */
97 CharFrontend server_chr; /* without interrupts */
98
99 /* registers */
100 uint32_t intrmask;
101 uint32_t intrstatus;
102 int vm_id;
103
104 /* BARs */
105 MemoryRegion ivshmem_mmio; /* BAR 0 (registers) */
106 MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */
107 MemoryRegion server_bar2; /* used with server_chr */
108
109 /* interrupt support */
110 Peer *peers;
111 int nb_peers; /* space in @peers[] */
112 uint32_t vectors;
113 MSIVector *msi_vectors;
114 uint64_t msg_buf; /* buffer for receiving server messages */
115 int msg_buffered_bytes; /* #bytes in @msg_buf */
116
117 /* migration stuff */
118 OnOffAuto master;
119 Error *migration_blocker;
120 };
121
122 /* registers for the Inter-VM shared memory device */
123 enum ivshmem_registers {
124 INTRMASK = 0,
125 INTRSTATUS = 4,
126 IVPOSITION = 8,
127 DOORBELL = 12,
128 };
129
130 static inline uint32_t ivshmem_has_feature(IVShmemState *ivs,
131 unsigned int feature) {
132 return (ivs->features & (1 << feature));
133 }
134
135 static inline bool ivshmem_is_master(IVShmemState *s)
136 {
137 assert(s->master != ON_OFF_AUTO_AUTO);
138 return s->master == ON_OFF_AUTO_ON;
139 }
140
141 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
142 {
143 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val);
144
145 s->intrmask = val;
146 }
147
148 static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
149 {
150 uint32_t ret = s->intrmask;
151
152 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret);
153 return ret;
154 }
155
156 static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
157 {
158 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val);
159
160 s->intrstatus = val;
161 }
162
163 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
164 {
165 uint32_t ret = s->intrstatus;
166
167 /* reading ISR clears all interrupts */
168 s->intrstatus = 0;
169 return ret;
170 }
171
172 static void ivshmem_io_write(void *opaque, hwaddr addr,
173 uint64_t val, unsigned size)
174 {
175 IVShmemState *s = opaque;
176
177 uint16_t dest = val >> 16;
178 uint16_t vector = val & 0xff;
179
180 addr &= 0xfc;
181
182 IVSHMEM_DPRINTF("writing to addr " HWADDR_FMT_plx "\n", addr);
183 switch (addr)
184 {
185 case INTRMASK:
186 ivshmem_IntrMask_write(s, val);
187 break;
188
189 case INTRSTATUS:
190 ivshmem_IntrStatus_write(s, val);
191 break;
192
193 case DOORBELL:
194 /* check that dest VM ID is reasonable */
195 if (dest >= s->nb_peers) {
196 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
197 break;
198 }
199
200 /* check doorbell range */
201 if (vector < s->peers[dest].nb_eventfds) {
202 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
203 event_notifier_set(&s->peers[dest].eventfds[vector]);
204 } else {
205 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
206 vector, dest);
207 }
208 break;
209 default:
210 IVSHMEM_DPRINTF("Unhandled write " HWADDR_FMT_plx "\n", addr);
211 }
212 }
213
214 static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
215 unsigned size)
216 {
217
218 IVShmemState *s = opaque;
219 uint32_t ret;
220
221 switch (addr)
222 {
223 case INTRMASK:
224 ret = ivshmem_IntrMask_read(s);
225 break;
226
227 case INTRSTATUS:
228 ret = ivshmem_IntrStatus_read(s);
229 break;
230
231 case IVPOSITION:
232 ret = s->vm_id;
233 break;
234
235 default:
236 IVSHMEM_DPRINTF("why are we reading " HWADDR_FMT_plx "\n", addr);
237 ret = 0;
238 }
239
240 return ret;
241 }
242
243 static const MemoryRegionOps ivshmem_mmio_ops = {
244 .read = ivshmem_io_read,
245 .write = ivshmem_io_write,
246 .endianness = DEVICE_LITTLE_ENDIAN,
247 .impl = {
248 .min_access_size = 4,
249 .max_access_size = 4,
250 },
251 };
252
253 static void ivshmem_vector_notify(void *opaque)
254 {
255 MSIVector *entry = opaque;
256 PCIDevice *pdev = entry->pdev;
257 IVShmemState *s = IVSHMEM_COMMON(pdev);
258 int vector = entry - s->msi_vectors;
259 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
260
261 if (!event_notifier_test_and_clear(n)) {
262 return;
263 }
264
265 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
266 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
267 if (msix_enabled(pdev)) {
268 msix_notify(pdev, vector);
269 }
270 } else {
271 ivshmem_IntrStatus_write(s, 1);
272 }
273 }
274
275 static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
276 MSIMessage msg)
277 {
278 IVShmemState *s = IVSHMEM_COMMON(dev);
279 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
280 MSIVector *v = &s->msi_vectors[vector];
281 int ret;
282
283 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
284 if (!v->pdev) {
285 error_report("ivshmem: vector %d route does not exist", vector);
286 return -EINVAL;
287 }
288 assert(!v->unmasked);
289
290 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
291 if (ret < 0) {
292 return ret;
293 }
294 kvm_irqchip_commit_routes(kvm_state);
295
296 ret = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
297 if (ret < 0) {
298 return ret;
299 }
300 v->unmasked = true;
301
302 return 0;
303 }
304
305 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
306 {
307 IVShmemState *s = IVSHMEM_COMMON(dev);
308 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
309 MSIVector *v = &s->msi_vectors[vector];
310 int ret;
311
312 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
313 if (!v->pdev) {
314 error_report("ivshmem: vector %d route does not exist", vector);
315 return;
316 }
317 assert(v->unmasked);
318
319 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, v->virq);
320 if (ret < 0) {
321 error_report("remove_irqfd_notifier_gsi failed");
322 return;
323 }
324 v->unmasked = false;
325 }
326
327 static void ivshmem_vector_poll(PCIDevice *dev,
328 unsigned int vector_start,
329 unsigned int vector_end)
330 {
331 IVShmemState *s = IVSHMEM_COMMON(dev);
332 unsigned int vector;
333
334 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
335
336 vector_end = MIN(vector_end, s->vectors);
337
338 for (vector = vector_start; vector < vector_end; vector++) {
339 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
340
341 if (!msix_is_masked(dev, vector)) {
342 continue;
343 }
344
345 if (event_notifier_test_and_clear(notifier)) {
346 msix_set_pending(dev, vector);
347 }
348 }
349 }
350
351 static void watch_vector_notifier(IVShmemState *s, EventNotifier *n,
352 int vector)
353 {
354 int eventfd = event_notifier_get_fd(n);
355
356 assert(!s->msi_vectors[vector].pdev);
357 s->msi_vectors[vector].pdev = PCI_DEVICE(s);
358
359 qemu_set_fd_handler(eventfd, ivshmem_vector_notify,
360 NULL, &s->msi_vectors[vector]);
361 }
362
363 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
364 {
365 memory_region_add_eventfd(&s->ivshmem_mmio,
366 DOORBELL,
367 4,
368 true,
369 (posn << 16) | i,
370 &s->peers[posn].eventfds[i]);
371 }
372
373 static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i)
374 {
375 memory_region_del_eventfd(&s->ivshmem_mmio,
376 DOORBELL,
377 4,
378 true,
379 (posn << 16) | i,
380 &s->peers[posn].eventfds[i]);
381 }
382
383 static void close_peer_eventfds(IVShmemState *s, int posn)
384 {
385 int i, n;
386
387 assert(posn >= 0 && posn < s->nb_peers);
388 n = s->peers[posn].nb_eventfds;
389
390 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
391 memory_region_transaction_begin();
392 for (i = 0; i < n; i++) {
393 ivshmem_del_eventfd(s, posn, i);
394 }
395 memory_region_transaction_commit();
396 }
397
398 for (i = 0; i < n; i++) {
399 event_notifier_cleanup(&s->peers[posn].eventfds[i]);
400 }
401
402 g_free(s->peers[posn].eventfds);
403 s->peers[posn].nb_eventfds = 0;
404 }
405
406 static void resize_peers(IVShmemState *s, int nb_peers)
407 {
408 int old_nb_peers = s->nb_peers;
409 int i;
410
411 assert(nb_peers > old_nb_peers);
412 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers);
413
414 s->peers = g_renew(Peer, s->peers, nb_peers);
415 s->nb_peers = nb_peers;
416
417 for (i = old_nb_peers; i < nb_peers; i++) {
418 s->peers[i].eventfds = g_new0(EventNotifier, s->vectors);
419 s->peers[i].nb_eventfds = 0;
420 }
421 }
422
423 static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector,
424 Error **errp)
425 {
426 PCIDevice *pdev = PCI_DEVICE(s);
427 AccelRouteChange c;
428 int ret;
429
430 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
431 assert(!s->msi_vectors[vector].pdev);
432
433 c = accel_irqchip_begin_route_changes();
434 ret = kvm_irqchip_add_msi_route(&c, vector, pdev);
435 if (ret < 0) {
436 error_setg(errp, "kvm_irqchip_add_msi_route failed");
437 return;
438 }
439 accel_irqchip_commit_route_changes(&c);
440
441 s->msi_vectors[vector].virq = ret;
442 s->msi_vectors[vector].pdev = pdev;
443 }
444
445 static bool setup_interrupt(IVShmemState *s, int vector, Error **errp)
446 {
447 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
448 bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
449 ivshmem_has_feature(s, IVSHMEM_MSI);
450 PCIDevice *pdev = PCI_DEVICE(s);
451 Error *err = NULL;
452 int ret;
453
454 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
455
456 if (!with_irqfd) {
457 IVSHMEM_DPRINTF("with eventfd\n");
458 watch_vector_notifier(s, n, vector);
459 } else if (msix_enabled(pdev)) {
460 IVSHMEM_DPRINTF("with irqfd\n");
461 ivshmem_add_kvm_msi_virq(s, vector, &err);
462 if (err) {
463 error_propagate(errp, err);
464 return false;
465 }
466
467 if (!msix_is_masked(pdev, vector)) {
468 ret = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
469 s->msi_vectors[vector].virq);
470 if (ret < 0) {
471 error_setg(errp, "Failed to configure irqfd notifier");
472 return false;
473 }
474 }
475 } else {
476 /* it will be delayed until msix is enabled, in write_config */
477 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
478 }
479 return true;
480 }
481
482 static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)
483 {
484 struct stat buf;
485 size_t size;
486
487 if (fd < 0) {
488 error_setg(errp, "server didn't provide fd with shared memory message");
489 return;
490 }
491
492 if (s->ivshmem_bar2) {
493 error_setg(errp, "server sent unexpected shared memory message");
494 close(fd);
495 return;
496 }
497
498 if (fstat(fd, &buf) < 0) {
499 error_setg_errno(errp, errno,
500 "can't determine size of shared memory sent by server");
501 close(fd);
502 return;
503 }
504
505 size = buf.st_size;
506
507 /* mmap the region and map into the BAR2 */
508 if (!memory_region_init_ram_from_fd(&s->server_bar2, OBJECT(s),
509 "ivshmem.bar2", size, RAM_SHARED,
510 fd, 0, errp)) {
511 return;
512 }
513
514 s->ivshmem_bar2 = &s->server_bar2;
515 }
516
517 static void process_msg_disconnect(IVShmemState *s, uint16_t posn,
518 Error **errp)
519 {
520 IVSHMEM_DPRINTF("posn %d has gone away\n", posn);
521 if (posn >= s->nb_peers || posn == s->vm_id) {
522 error_setg(errp, "invalid peer %d", posn);
523 return;
524 }
525 close_peer_eventfds(s, posn);
526 }
527
528 static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd,
529 Error **errp)
530 {
531 Peer *peer = &s->peers[posn];
532 int vector;
533
534 /*
535 * The N-th connect message for this peer comes with the file
536 * descriptor for vector N-1. Count messages to find the vector.
537 */
538 if (peer->nb_eventfds >= s->vectors) {
539 error_setg(errp, "Too many eventfd received, device has %d vectors",
540 s->vectors);
541 close(fd);
542 return;
543 }
544 vector = peer->nb_eventfds++;
545
546 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd);
547 event_notifier_init_fd(&peer->eventfds[vector], fd);
548
549 /* msix/irqfd poll non block */
550 if (!qemu_set_blocking(fd, false, errp)) {
551 close(fd);
552 return;
553 }
554
555 if (posn == s->vm_id) {
556 setup_interrupt(s, vector, errp);
557 /* TODO do we need to handle the error? */
558 }
559
560 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
561 ivshmem_add_eventfd(s, posn, vector);
562 }
563 }
564
565 static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp)
566 {
567 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
568
569 if (msg < -1 || msg > IVSHMEM_MAX_PEERS) {
570 error_setg(errp, "server sent invalid message %" PRId64, msg);
571 if (fd >= 0) {
572 close(fd);
573 }
574 return;
575 }
576
577 if (msg == -1) {
578 process_msg_shmem(s, fd, errp);
579 return;
580 }
581
582 if (msg >= s->nb_peers) {
583 resize_peers(s, msg + 1);
584 }
585
586 if (fd >= 0) {
587 process_msg_connect(s, msg, fd, errp);
588 } else {
589 process_msg_disconnect(s, msg, errp);
590 }
591 }
592
593 static int ivshmem_can_receive(void *opaque)
594 {
595 IVShmemState *s = opaque;
596
597 assert(s->msg_buffered_bytes < sizeof(s->msg_buf));
598 return sizeof(s->msg_buf) - s->msg_buffered_bytes;
599 }
600
601 static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
602 {
603 IVShmemState *s = opaque;
604 Error *err = NULL;
605 int fd;
606 int64_t msg;
607
608 assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf));
609 memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size);
610 s->msg_buffered_bytes += size;
611 if (s->msg_buffered_bytes < sizeof(s->msg_buf)) {
612 return;
613 }
614 msg = le64_to_cpu(s->msg_buf);
615 s->msg_buffered_bytes = 0;
616
617 fd = qemu_chr_fe_get_msgfd(&s->server_chr);
618
619 process_msg(s, msg, fd, &err);
620 if (err) {
621 error_report_err(err);
622 }
623 }
624
625 static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp)
626 {
627 int64_t msg;
628 int n, ret;
629
630 n = 0;
631 do {
632 ret = qemu_chr_fe_read_all(&s->server_chr, (uint8_t *)&msg + n,
633 sizeof(msg) - n);
634 if (ret < 0) {
635 if (ret == -EINTR) {
636 continue;
637 }
638 error_setg_errno(errp, -ret, "read from server failed");
639 return INT64_MIN;
640 }
641 n += ret;
642 } while (n < sizeof(msg));
643
644 *pfd = qemu_chr_fe_get_msgfd(&s->server_chr);
645 return le64_to_cpu(msg);
646 }
647
648 static void ivshmem_recv_setup(IVShmemState *s, Error **errp)
649 {
650 Error *err = NULL;
651 int64_t msg;
652 int fd;
653
654 msg = ivshmem_recv_msg(s, &fd, &err);
655 if (err) {
656 error_propagate(errp, err);
657 return;
658 }
659 if (msg != IVSHMEM_PROTOCOL_VERSION) {
660 error_setg(errp, "server sent version %" PRId64 ", expecting %d",
661 msg, IVSHMEM_PROTOCOL_VERSION);
662 return;
663 }
664 if (fd != -1) {
665 error_setg(errp, "server sent invalid version message");
666 return;
667 }
668
669 /*
670 * ivshmem-server sends the remaining initial messages in a fixed
671 * order, but the device has always accepted them in any order.
672 * Stay as compatible as practical, just in case people use
673 * servers that behave differently.
674 */
675
676 /*
677 * ivshmem_device_spec.txt has always required the ID message
678 * right here, and ivshmem-server has always complied. However,
679 * older versions of the device accepted it out of order, but
680 * broke when an interrupt setup message arrived before it.
681 */
682 msg = ivshmem_recv_msg(s, &fd, &err);
683 if (err) {
684 error_propagate(errp, err);
685 return;
686 }
687 if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) {
688 error_setg(errp, "server sent invalid ID message");
689 return;
690 }
691 s->vm_id = msg;
692
693 /*
694 * Receive more messages until we got shared memory.
695 */
696 do {
697 msg = ivshmem_recv_msg(s, &fd, &err);
698 if (err) {
699 error_propagate(errp, err);
700 return;
701 }
702 process_msg(s, msg, fd, &err);
703 if (err) {
704 error_propagate(errp, err);
705 return;
706 }
707 } while (msg != -1);
708
709 /*
710 * This function must either map the shared memory or fail. The
711 * loop above ensures that: it terminates normally only after it
712 * successfully processed the server's shared memory message.
713 * Assert that actually mapped the shared memory:
714 */
715 assert(s->ivshmem_bar2);
716 }
717
718 /* Select the MSI-X vectors used by device.
719 * ivshmem maps events to vectors statically, so
720 * we just enable all vectors on init and after reset. */
721 static void ivshmem_msix_vector_use(IVShmemState *s)
722 {
723 PCIDevice *d = PCI_DEVICE(s);
724 int i;
725
726 for (i = 0; i < s->vectors; i++) {
727 msix_vector_use(d, i);
728 }
729 }
730
731 static void ivshmem_disable_irqfd(IVShmemState *s);
732
733 static void ivshmem_reset(DeviceState *d)
734 {
735 IVShmemState *s = IVSHMEM_COMMON(d);
736
737 ivshmem_disable_irqfd(s);
738
739 s->intrstatus = 0;
740 s->intrmask = 0;
741 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
742 ivshmem_msix_vector_use(s);
743 }
744 }
745
746 static int ivshmem_setup_interrupts(IVShmemState *s, Error **errp)
747 {
748 /* allocate QEMU callback data for receiving interrupts */
749 s->msi_vectors = g_new0(MSIVector, s->vectors);
750
751 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
752 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1, errp)) {
753 return -1;
754 }
755
756 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
757 ivshmem_msix_vector_use(s);
758 }
759
760 return 0;
761 }
762
763 static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
764 {
765 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
766
767 if (s->msi_vectors[vector].pdev == NULL) {
768 return;
769 }
770
771 /* it was cleaned when masked in the frontend. */
772 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
773
774 s->msi_vectors[vector].pdev = NULL;
775 }
776
777 static void ivshmem_enable_irqfd(IVShmemState *s)
778 {
779 PCIDevice *pdev = PCI_DEVICE(s);
780 int i;
781
782 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
783 Error *err = NULL;
784
785 ivshmem_add_kvm_msi_virq(s, i, &err);
786 if (err) {
787 error_report_err(err);
788 goto undo;
789 }
790 }
791
792 if (msix_set_vector_notifiers(pdev,
793 ivshmem_vector_unmask,
794 ivshmem_vector_mask,
795 ivshmem_vector_poll)) {
796 error_report("ivshmem: msix_set_vector_notifiers failed");
797 goto undo;
798 }
799 return;
800
801 undo:
802 while (--i >= 0) {
803 ivshmem_remove_kvm_msi_virq(s, i);
804 }
805 }
806
807 static void ivshmem_disable_irqfd(IVShmemState *s)
808 {
809 PCIDevice *pdev = PCI_DEVICE(s);
810 int i;
811
812 if (!pdev->msix_vector_use_notifier) {
813 return;
814 }
815
816 msix_unset_vector_notifiers(pdev);
817
818 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
819 /*
820 * MSI-X is already disabled here so msix_unset_vector_notifiers()
821 * didn't call our release notifier. Do it now to keep our masks and
822 * unmasks balanced.
823 */
824 if (s->msi_vectors[i].unmasked) {
825 ivshmem_vector_mask(pdev, i);
826 }
827 ivshmem_remove_kvm_msi_virq(s, i);
828 }
829
830 }
831
832 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
833 uint32_t val, int len)
834 {
835 IVShmemState *s = IVSHMEM_COMMON(pdev);
836 int is_enabled, was_enabled = msix_enabled(pdev);
837
838 pci_default_write_config(pdev, address, val, len);
839 is_enabled = msix_enabled(pdev);
840
841 if (kvm_msi_via_irqfd_enabled()) {
842 if (!was_enabled && is_enabled) {
843 ivshmem_enable_irqfd(s);
844 } else if (was_enabled && !is_enabled) {
845 ivshmem_disable_irqfd(s);
846 }
847 }
848 }
849
850 static void ivshmem_common_realize(PCIDevice *dev, Error **errp)
851 {
852 ERRP_GUARD();
853 IVShmemState *s = IVSHMEM_COMMON(dev);
854 Error *err = NULL;
855 uint8_t *pci_conf;
856
857 /* IRQFD requires MSI */
858 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
859 !ivshmem_has_feature(s, IVSHMEM_MSI)) {
860 error_setg(errp, "ioeventfd/irqfd requires MSI");
861 return;
862 }
863
864 pci_conf = dev->config;
865 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
866
867 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
868 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
869
870 /* region for registers*/
871 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
872 &s->ivshmem_mmio);
873
874 if (s->hostmem != NULL) {
875 IVSHMEM_DPRINTF("using hostmem\n");
876
877 s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem);
878 host_memory_backend_set_mapped(s->hostmem, true);
879 } else {
880 Chardev *chr = qemu_chr_fe_get_driver(&s->server_chr);
881 g_autofree char *filename = NULL;
882
883 assert(chr);
884 filename = qemu_chr_get_filename(chr);
885 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n", filename);
886
887 /* we allocate enough space for 16 peers and grow as needed */
888 resize_peers(s, 16);
889
890 /*
891 * Receive setup messages from server synchronously.
892 * Older versions did it asynchronously, but that creates a
893 * number of entertaining race conditions.
894 */
895 ivshmem_recv_setup(s, &err);
896 if (err) {
897 error_propagate(errp, err);
898 return;
899 }
900
901 if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) {
902 error_setg(errp,
903 "master must connect to the server before any peers");
904 return;
905 }
906
907 qemu_chr_fe_set_handlers(&s->server_chr, ivshmem_can_receive,
908 ivshmem_read, NULL, NULL, s, NULL, true);
909
910 if (ivshmem_setup_interrupts(s, errp) < 0) {
911 error_prepend(errp, "Failed to initialize interrupts: ");
912 return;
913 }
914 }
915
916 if (s->master == ON_OFF_AUTO_AUTO) {
917 s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
918 }
919
920 if (!ivshmem_is_master(s)) {
921 error_setg(&s->migration_blocker,
922 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
923 if (migrate_add_blocker(&s->migration_blocker, errp) < 0) {
924 return;
925 }
926 }
927
928 vmstate_register_ram(s->ivshmem_bar2, DEVICE(s));
929 pci_register_bar(PCI_DEVICE(s), 2,
930 PCI_BASE_ADDRESS_SPACE_MEMORY |
931 PCI_BASE_ADDRESS_MEM_PREFETCH |
932 PCI_BASE_ADDRESS_MEM_TYPE_64,
933 s->ivshmem_bar2);
934 }
935
936 static void ivshmem_exit(PCIDevice *dev)
937 {
938 IVShmemState *s = IVSHMEM_COMMON(dev);
939 int i;
940
941 qemu_chr_fe_set_handlers(&s->server_chr,
942 NULL, NULL, NULL, NULL, NULL, NULL, true);
943
944 migrate_del_blocker(&s->migration_blocker);
945
946 if (memory_region_is_mapped(s->ivshmem_bar2)) {
947 if (!s->hostmem) {
948 void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2);
949 int fd;
950
951 if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) {
952 error_report("Failed to munmap shared memory %s",
953 strerror(errno));
954 }
955
956 fd = memory_region_get_fd(s->ivshmem_bar2);
957 close(fd);
958 }
959
960 vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev));
961 }
962
963 if (s->hostmem) {
964 host_memory_backend_set_mapped(s->hostmem, false);
965 }
966
967 if (s->peers) {
968 for (i = 0; i < s->nb_peers; i++) {
969 close_peer_eventfds(s, i);
970 }
971 g_free(s->peers);
972 s->peers = NULL;
973 s->nb_peers = 0;
974 }
975
976 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
977 msix_uninit_exclusive_bar(dev);
978 }
979
980 g_free(s->msi_vectors);
981 s->msi_vectors = NULL;
982 }
983
984 static int ivshmem_pre_load(void *opaque)
985 {
986 IVShmemState *s = opaque;
987
988 if (!ivshmem_is_master(s)) {
989 error_report("'peer' devices are not migratable");
990 return -EINVAL;
991 }
992
993 return 0;
994 }
995
996 static int ivshmem_post_load(void *opaque, int version_id)
997 {
998 IVShmemState *s = opaque;
999
1000 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
1001 ivshmem_msix_vector_use(s);
1002 }
1003 return 0;
1004 }
1005
1006 static void ivshmem_common_class_init(ObjectClass *klass, const void *data)
1007 {
1008 DeviceClass *dc = DEVICE_CLASS(klass);
1009 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1010
1011 k->realize = ivshmem_common_realize;
1012 k->exit = ivshmem_exit;
1013 k->config_write = ivshmem_write_config;
1014 k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
1015 k->device_id = PCI_DEVICE_ID_IVSHMEM;
1016 k->class_id = PCI_CLASS_MEMORY_RAM;
1017 k->revision = 1;
1018 device_class_set_legacy_reset(dc, ivshmem_reset);
1019 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1020 dc->desc = "Inter-VM shared memory";
1021 }
1022
1023 static const TypeInfo ivshmem_common_info = {
1024 .name = TYPE_IVSHMEM_COMMON,
1025 .parent = TYPE_PCI_DEVICE,
1026 .instance_size = sizeof(IVShmemState),
1027 .abstract = true,
1028 .class_init = ivshmem_common_class_init,
1029 .interfaces = (const InterfaceInfo[]) {
1030 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
1031 { },
1032 },
1033 };
1034
1035 static const VMStateDescription ivshmem_plain_vmsd = {
1036 .name = TYPE_IVSHMEM_PLAIN,
1037 .version_id = 0,
1038 .minimum_version_id = 0,
1039 .pre_load = ivshmem_pre_load,
1040 .post_load = ivshmem_post_load,
1041 .fields = (const VMStateField[]) {
1042 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1043 VMSTATE_UINT32(intrstatus, IVShmemState),
1044 VMSTATE_UINT32(intrmask, IVShmemState),
1045 VMSTATE_END_OF_LIST()
1046 },
1047 };
1048
1049 static const Property ivshmem_plain_properties[] = {
1050 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1051 DEFINE_PROP_LINK("memdev", IVShmemState, hostmem, TYPE_MEMORY_BACKEND,
1052 HostMemoryBackend *),
1053 };
1054
1055 static void ivshmem_plain_realize(PCIDevice *dev, Error **errp)
1056 {
1057 IVShmemState *s = IVSHMEM_COMMON(dev);
1058
1059 if (!s->hostmem) {
1060 error_setg(errp, "You must specify a 'memdev'");
1061 return;
1062 } else if (host_memory_backend_is_mapped(s->hostmem)) {
1063 error_setg(errp, "can't use already busy memdev: %s",
1064 object_get_canonical_path_component(OBJECT(s->hostmem)));
1065 return;
1066 }
1067
1068 ivshmem_common_realize(dev, errp);
1069 }
1070
1071 static void ivshmem_plain_class_init(ObjectClass *klass, const void *data)
1072 {
1073 DeviceClass *dc = DEVICE_CLASS(klass);
1074 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1075
1076 k->realize = ivshmem_plain_realize;
1077 device_class_set_props(dc, ivshmem_plain_properties);
1078 dc->vmsd = &ivshmem_plain_vmsd;
1079 }
1080
1081 static const TypeInfo ivshmem_plain_info = {
1082 .name = TYPE_IVSHMEM_PLAIN,
1083 .parent = TYPE_IVSHMEM_COMMON,
1084 .instance_size = sizeof(IVShmemState),
1085 .class_init = ivshmem_plain_class_init,
1086 };
1087
1088 static const VMStateDescription ivshmem_doorbell_vmsd = {
1089 .name = TYPE_IVSHMEM_DOORBELL,
1090 .version_id = 0,
1091 .minimum_version_id = 0,
1092 .pre_load = ivshmem_pre_load,
1093 .post_load = ivshmem_post_load,
1094 .fields = (const VMStateField[]) {
1095 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1096 VMSTATE_MSIX(parent_obj, IVShmemState),
1097 VMSTATE_UINT32(intrstatus, IVShmemState),
1098 VMSTATE_UINT32(intrmask, IVShmemState),
1099 VMSTATE_END_OF_LIST()
1100 },
1101 };
1102
1103 static const Property ivshmem_doorbell_properties[] = {
1104 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
1105 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
1106 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
1107 true),
1108 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1109 };
1110
1111 static void ivshmem_doorbell_init(Object *obj)
1112 {
1113 IVShmemState *s = IVSHMEM_DOORBELL(obj);
1114
1115 s->features |= (1 << IVSHMEM_MSI);
1116 }
1117
1118 static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp)
1119 {
1120 IVShmemState *s = IVSHMEM_COMMON(dev);
1121
1122 if (!qemu_chr_fe_backend_connected(&s->server_chr)) {
1123 error_setg(errp, "You must specify a 'chardev'");
1124 return;
1125 }
1126
1127 ivshmem_common_realize(dev, errp);
1128 }
1129
1130 static void ivshmem_doorbell_class_init(ObjectClass *klass, const void *data)
1131 {
1132 DeviceClass *dc = DEVICE_CLASS(klass);
1133 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1134
1135 k->realize = ivshmem_doorbell_realize;
1136 device_class_set_props(dc, ivshmem_doorbell_properties);
1137 dc->vmsd = &ivshmem_doorbell_vmsd;
1138 }
1139
1140 static const TypeInfo ivshmem_doorbell_info = {
1141 .name = TYPE_IVSHMEM_DOORBELL,
1142 .parent = TYPE_IVSHMEM_COMMON,
1143 .instance_size = sizeof(IVShmemState),
1144 .instance_init = ivshmem_doorbell_init,
1145 .class_init = ivshmem_doorbell_class_init,
1146 };
1147
1148 static void ivshmem_register_types(void)
1149 {
1150 type_register_static(&ivshmem_common_info);
1151 type_register_static(&ivshmem_plain_info);
1152 type_register_static(&ivshmem_doorbell_info);
1153 }
1154
1155 type_init(ivshmem_register_types)