| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| 2 | |
| 3 | #include "qemu/osdep.h" |
| 4 | #include "qemu/thread.h" |
| 5 | |
| 6 | /* |
| 7 | * Valid transitions: |
| 8 | * - FREE -> SET (qemu_event_set) |
| 9 | * - BUSY -> SET (qemu_event_set) |
| 10 | * - SET -> FREE (qemu_event_reset) |
| 11 | * - FREE -> BUSY (qemu_event_wait) |
| 12 | * |
| 13 | * With futex, the waking and blocking operations follow |
| 14 | * BUSY -> SET and FREE -> BUSY, respectively. |
| 15 | * |
| 16 | * Without futex, BUSY -> SET and FREE -> BUSY never happen. Instead, the waking |
| 17 | * operation follows FREE -> SET and the blocking operation will happen in |
| 18 | * qemu_event_wait() if the event is not SET. |
| 19 | * |
| 20 | * SET->BUSY does not happen (it can be observed from the outside but |
| 21 | * it really is SET->FREE->BUSY). |
| 22 | * |
| 23 | * busy->free provably cannot happen; to enforce it, the set->free transition |
| 24 | * is done with an OR, which becomes a no-op if the event has concurrently |
| 25 | * transitioned to free or busy. |
| 26 | */ |
| 27 | |
| 28 | #define EV_SET 0 |
| 29 | #define EV_FREE 1 |
| 30 | #define EV_BUSY -1 |
| 31 | |
| 32 | void qemu_event_init(QemuEvent *ev, bool init) |
| 33 | { |
| 34 | #ifndef HAVE_FUTEX |
| 35 | pthread_mutex_init(&ev->lock, NULL); |
| 36 | pthread_cond_init(&ev->cond, NULL); |
| 37 | #endif |
| 38 | |
| 39 | ev->value = (init ? EV_SET : EV_FREE); |
| 40 | ev->initialized = true; |
| 41 | } |
| 42 | |
| 43 | void qemu_event_destroy(QemuEvent *ev) |
| 44 | { |
| 45 | assert(ev->initialized); |
| 46 | ev->initialized = false; |
| 47 | #ifndef HAVE_FUTEX |
| 48 | pthread_mutex_destroy(&ev->lock); |
| 49 | pthread_cond_destroy(&ev->cond); |
| 50 | #endif |
| 51 | } |
| 52 | |
| 53 | void qemu_event_set(QemuEvent *ev) |
| 54 | { |
| 55 | assert(ev->initialized); |
| 56 | |
| 57 | #ifdef HAVE_FUTEX |
| 58 | /* |
| 59 | * Pairs with both qemu_event_reset() and qemu_event_wait(). |
| 60 | * |
| 61 | * qemu_event_set has release semantics, but because it *loads* |
| 62 | * ev->value we need a full memory barrier here. |
| 63 | */ |
| 64 | smp_mb(); |
| 65 | if (qatomic_read(&ev->value) != EV_SET) { |
| 66 | int old = qatomic_xchg(&ev->value, EV_SET); |
| 67 | |
| 68 | /* Pairs with memory barrier in kernel futex_wait system call. */ |
| 69 | smp_mb__after_rmw(); |
| 70 | if (old == EV_BUSY) { |
| 71 | /* There were waiters, wake them up. */ |
| 72 | qemu_futex_wake_all(ev); |
| 73 | } |
| 74 | } |
| 75 | #else |
| 76 | pthread_mutex_lock(&ev->lock); |
| 77 | /* Pairs with qemu_event_reset()'s load acquire. */ |
| 78 | qatomic_store_release(&ev->value, EV_SET); |
| 79 | pthread_cond_broadcast(&ev->cond); |
| 80 | pthread_mutex_unlock(&ev->lock); |
| 81 | #endif |
| 82 | } |
| 83 | |
| 84 | void qemu_event_reset(QemuEvent *ev) |
| 85 | { |
| 86 | assert(ev->initialized); |
| 87 | |
| 88 | #ifdef HAVE_FUTEX |
| 89 | /* |
| 90 | * If there was a concurrent reset (or even reset+wait), |
| 91 | * do nothing. Otherwise change EV_SET->EV_FREE. |
| 92 | */ |
| 93 | qatomic_or(&ev->value, EV_FREE); |
| 94 | |
| 95 | /* |
| 96 | * Order reset before checking the condition in the caller. |
| 97 | * Pairs with the first memory barrier in qemu_event_set(). |
| 98 | */ |
| 99 | smp_mb__after_rmw(); |
| 100 | #else |
| 101 | /* |
| 102 | * If futexes are not available, there are no EV_FREE->EV_BUSY |
| 103 | * transitions because wakeups are done entirely through the |
| 104 | * condition variable. Since qatomic_set() only writes EV_FREE, |
| 105 | * the load seems useless but in reality, the acquire synchronizes |
| 106 | * with qemu_event_set()'s store release: if qemu_event_reset() |
| 107 | * sees EV_SET here, then the caller will certainly see a |
| 108 | * successful condition and skip qemu_event_wait(): |
| 109 | * |
| 110 | * done = 1; if (done == 0) |
| 111 | * qemu_event_set() { qemu_event_reset() { |
| 112 | * lock(); |
| 113 | * ev->value = EV_SET -----> load ev->value |
| 114 | * ev->value = old value | EV_FREE |
| 115 | * cond_broadcast() |
| 116 | * unlock(); } |
| 117 | * } if (done == 0) |
| 118 | * // qemu_event_wait() not called |
| 119 | */ |
| 120 | qatomic_set(&ev->value, qatomic_load_acquire(&ev->value) | EV_FREE); |
| 121 | #endif |
| 122 | } |
| 123 | |
| 124 | void qemu_event_wait(QemuEvent *ev) |
| 125 | { |
| 126 | assert(ev->initialized); |
| 127 | |
| 128 | #ifdef HAVE_FUTEX |
| 129 | while (true) { |
| 130 | /* |
| 131 | * qemu_event_wait must synchronize with qemu_event_set even if it does |
| 132 | * not go down the slow path, so this load-acquire is needed that |
| 133 | * synchronizes with the first memory barrier in qemu_event_set(). |
| 134 | */ |
| 135 | unsigned value = qatomic_load_acquire(&ev->value); |
| 136 | if (value == EV_SET) { |
| 137 | break; |
| 138 | } |
| 139 | |
| 140 | if (value == EV_FREE) { |
| 141 | /* |
| 142 | * Leave the event reset and tell qemu_event_set that there are |
| 143 | * waiters. No need to retry, because there cannot be a concurrent |
| 144 | * busy->free transition. After the CAS, the event will be either |
| 145 | * set or busy. |
| 146 | * |
| 147 | * This cmpxchg doesn't have particular ordering requirements if it |
| 148 | * succeeds (moving the store earlier can only cause |
| 149 | * qemu_event_set() to issue _more_ wakeups), the failing case needs |
| 150 | * acquire semantics like the load above. |
| 151 | */ |
| 152 | if (qatomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) { |
| 153 | break; |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | /* |
| 158 | * This is the final check for a concurrent set, so it does need |
| 159 | * a smp_mb() pairing with the second barrier of qemu_event_set(). |
| 160 | * The barrier is inside the FUTEX_WAIT system call. |
| 161 | */ |
| 162 | qemu_futex_wait(ev, EV_BUSY); |
| 163 | } |
| 164 | #else |
| 165 | pthread_mutex_lock(&ev->lock); |
| 166 | while (qatomic_read(&ev->value) != EV_SET) { |
| 167 | pthread_cond_wait(&ev->cond, &ev->lock); |
| 168 | } |
| 169 | pthread_mutex_unlock(&ev->lock); |
| 170 | #endif |
| 171 | } |