| 1 | /* |
| 2 | * Data plane event loop |
| 3 | * |
| 4 | * Copyright (c) 2003-2008 Fabrice Bellard |
| 5 | * Copyright (c) 2009-2017 QEMU contributors |
| 6 | * |
| 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 8 | * of this software and associated documentation files (the "Software"), to deal |
| 9 | * in the Software without restriction, including without limitation the rights |
| 10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 11 | * copies of the Software, and to permit persons to whom the Software is |
| 12 | * furnished to do so, subject to the following conditions: |
| 13 | * |
| 14 | * The above copyright notice and this permission notice shall be included in |
| 15 | * all copies or substantial portions of the Software. |
| 16 | * |
| 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 23 | * THE SOFTWARE. |
| 24 | */ |
| 25 | |
| 26 | #include "qemu/osdep.h" |
| 27 | #include "qapi/error.h" |
| 28 | #include "qemu/aio.h" |
| 29 | #include "block/thread-pool.h" |
| 30 | #include "block/graph-lock.h" |
| 31 | #include "qemu/main-loop.h" |
| 32 | #include "qemu/mem-reentrancy.h" |
| 33 | #include "qemu/atomic.h" |
| 34 | #include "qemu/lockcnt.h" |
| 35 | #include "qemu/rcu_queue.h" |
| 36 | #include "block/raw-aio.h" |
| 37 | #include "qemu/coroutine_int.h" |
| 38 | #include "qemu/coroutine-tls.h" |
| 39 | #include "exec/icount.h" |
| 40 | #include "trace.h" |
| 41 | |
| 42 | /***********************************************************/ |
| 43 | /* bottom halves (can be seen as timers which expire ASAP) */ |
| 44 | |
| 45 | /* QEMUBH::flags values */ |
| 46 | enum { |
| 47 | /* Already enqueued and waiting for aio_bh_poll() */ |
| 48 | BH_PENDING = (1 << 0), |
| 49 | |
| 50 | /* Invoke the callback */ |
| 51 | BH_SCHEDULED = (1 << 1), |
| 52 | |
| 53 | /* Delete without invoking callback */ |
| 54 | BH_DELETED = (1 << 2), |
| 55 | |
| 56 | /* Delete after invoking callback */ |
| 57 | BH_ONESHOT = (1 << 3), |
| 58 | |
| 59 | /* Schedule periodically when the event loop is idle */ |
| 60 | BH_IDLE = (1 << 4), |
| 61 | }; |
| 62 | |
| 63 | struct QEMUBH { |
| 64 | AioContext *ctx; |
| 65 | const char *name; |
| 66 | QEMUBHFunc *cb; |
| 67 | void *opaque; |
| 68 | QSLIST_ENTRY(QEMUBH) next; |
| 69 | unsigned flags; |
| 70 | MemReentrancyGuard *reentrancy_guard; |
| 71 | }; |
| 72 | |
| 73 | /* Called concurrently from any thread */ |
| 74 | static void aio_bh_enqueue(QEMUBH *bh, unsigned new_flags) |
| 75 | { |
| 76 | AioContext *ctx = bh->ctx; |
| 77 | unsigned old_flags; |
| 78 | |
| 79 | /* |
| 80 | * Synchronizes with atomic_fetch_and() in aio_bh_dequeue(), ensuring that |
| 81 | * insertion starts after BH_PENDING is set. |
| 82 | */ |
| 83 | old_flags = qatomic_fetch_or(&bh->flags, BH_PENDING | new_flags); |
| 84 | |
| 85 | if (!(old_flags & BH_PENDING)) { |
| 86 | /* |
| 87 | * At this point the bottom half becomes visible to aio_bh_poll(). |
| 88 | * This insertion thus synchronizes with QSLIST_MOVE_ATOMIC in |
| 89 | * aio_bh_poll(), ensuring that: |
| 90 | * 1. any writes needed by the callback are visible from the callback |
| 91 | * after aio_bh_dequeue() returns bh. |
| 92 | * 2. ctx is loaded before the callback has a chance to execute and bh |
| 93 | * could be freed. |
| 94 | */ |
| 95 | QSLIST_INSERT_HEAD_ATOMIC(&ctx->bh_list, bh, next); |
| 96 | } |
| 97 | |
| 98 | aio_notify(ctx); |
| 99 | if (unlikely(icount_enabled())) { |
| 100 | /* |
| 101 | * Workaround for record/replay. |
| 102 | * vCPU execution should be suspended when new BH is set. |
| 103 | * This is needed to avoid guest timeouts caused |
| 104 | * by the long cycles of the execution. |
| 105 | */ |
| 106 | icount_notify_exit(); |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | /* Only called from aio_bh_poll() and aio_ctx_finalize() */ |
| 111 | static QEMUBH *aio_bh_dequeue(BHList *head, unsigned *flags) |
| 112 | { |
| 113 | QEMUBH *bh = QSLIST_FIRST_RCU(head); |
| 114 | |
| 115 | if (!bh) { |
| 116 | return NULL; |
| 117 | } |
| 118 | |
| 119 | QSLIST_REMOVE_HEAD(head, next); |
| 120 | |
| 121 | /* |
| 122 | * Synchronizes with qatomic_fetch_or() in aio_bh_enqueue(), ensuring that |
| 123 | * the removal finishes before BH_PENDING is reset. |
| 124 | */ |
| 125 | *flags = qatomic_fetch_and(&bh->flags, |
| 126 | ~(BH_PENDING | BH_SCHEDULED | BH_IDLE)); |
| 127 | return bh; |
| 128 | } |
| 129 | |
| 130 | void aio_bh_schedule_oneshot_full(AioContext *ctx, QEMUBHFunc *cb, |
| 131 | void *opaque, const char *name) |
| 132 | { |
| 133 | QEMUBH *bh; |
| 134 | bh = g_new(QEMUBH, 1); |
| 135 | *bh = (QEMUBH){ |
| 136 | .ctx = ctx, |
| 137 | .cb = cb, |
| 138 | .opaque = opaque, |
| 139 | .name = name, |
| 140 | }; |
| 141 | aio_bh_enqueue(bh, BH_SCHEDULED | BH_ONESHOT); |
| 142 | } |
| 143 | |
| 144 | QEMUBH *aio_bh_new_full(AioContext *ctx, QEMUBHFunc *cb, void *opaque, |
| 145 | const char *name, MemReentrancyGuard *reentrancy_guard) |
| 146 | { |
| 147 | QEMUBH *bh; |
| 148 | bh = g_new(QEMUBH, 1); |
| 149 | *bh = (QEMUBH){ |
| 150 | .ctx = ctx, |
| 151 | .cb = cb, |
| 152 | .opaque = opaque, |
| 153 | .name = name, |
| 154 | .reentrancy_guard = reentrancy_guard, |
| 155 | }; |
| 156 | return bh; |
| 157 | } |
| 158 | |
| 159 | void aio_bh_call(QEMUBH *bh) |
| 160 | { |
| 161 | bool last_engaged_in_io = false; |
| 162 | |
| 163 | /* Make a copy of the guard-pointer as cb may free the bh */ |
| 164 | MemReentrancyGuard *reentrancy_guard = bh->reentrancy_guard; |
| 165 | if (reentrancy_guard) { |
| 166 | last_engaged_in_io = reentrancy_guard->engaged_in_io; |
| 167 | if (reentrancy_guard->engaged_in_io) { |
| 168 | trace_reentrant_aio(bh->ctx, bh->name); |
| 169 | } |
| 170 | reentrancy_guard->engaged_in_io = true; |
| 171 | } |
| 172 | |
| 173 | bh->cb(bh->opaque); |
| 174 | |
| 175 | if (reentrancy_guard) { |
| 176 | reentrancy_guard->engaged_in_io = last_engaged_in_io; |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | /* Multiple occurrences of aio_bh_poll cannot be called concurrently. */ |
| 181 | int aio_bh_poll(AioContext *ctx) |
| 182 | { |
| 183 | BHListSlice slice; |
| 184 | BHListSlice *s; |
| 185 | int ret = 0; |
| 186 | |
| 187 | /* Synchronizes with QSLIST_INSERT_HEAD_ATOMIC in aio_bh_enqueue(). */ |
| 188 | QSLIST_MOVE_ATOMIC(&slice.bh_list, &ctx->bh_list); |
| 189 | |
| 190 | /* |
| 191 | * GCC13 [-Werror=dangling-pointer=] complains that the local variable |
| 192 | * 'slice' is being stored in the global 'ctx->bh_slice_list' but the |
| 193 | * list is emptied before this function returns. |
| 194 | */ |
| 195 | #if !defined(__clang__) |
| 196 | #pragma GCC diagnostic push |
| 197 | #pragma GCC diagnostic ignored "-Wpragmas" |
| 198 | #pragma GCC diagnostic ignored "-Wdangling-pointer=" |
| 199 | #endif |
| 200 | QSIMPLEQ_INSERT_TAIL(&ctx->bh_slice_list, &slice, next); |
| 201 | #if !defined(__clang__) |
| 202 | #pragma GCC diagnostic pop |
| 203 | #endif |
| 204 | |
| 205 | while ((s = QSIMPLEQ_FIRST(&ctx->bh_slice_list))) { |
| 206 | QEMUBH *bh; |
| 207 | unsigned flags; |
| 208 | |
| 209 | bh = aio_bh_dequeue(&s->bh_list, &flags); |
| 210 | if (!bh) { |
| 211 | QSIMPLEQ_REMOVE_HEAD(&ctx->bh_slice_list, next); |
| 212 | continue; |
| 213 | } |
| 214 | |
| 215 | if ((flags & (BH_SCHEDULED | BH_DELETED)) == BH_SCHEDULED) { |
| 216 | /* Idle BHs don't count as progress */ |
| 217 | if (!(flags & BH_IDLE)) { |
| 218 | ret = 1; |
| 219 | } |
| 220 | aio_bh_call(bh); |
| 221 | } |
| 222 | if (flags & (BH_DELETED | BH_ONESHOT)) { |
| 223 | g_free(bh); |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | return ret; |
| 228 | } |
| 229 | |
| 230 | void qemu_bh_schedule_idle(QEMUBH *bh) |
| 231 | { |
| 232 | aio_bh_enqueue(bh, BH_SCHEDULED | BH_IDLE); |
| 233 | } |
| 234 | |
| 235 | void qemu_bh_schedule(QEMUBH *bh) |
| 236 | { |
| 237 | aio_bh_enqueue(bh, BH_SCHEDULED); |
| 238 | } |
| 239 | |
| 240 | /* This func is async. |
| 241 | */ |
| 242 | void qemu_bh_cancel(QEMUBH *bh) |
| 243 | { |
| 244 | qatomic_and(&bh->flags, ~BH_SCHEDULED); |
| 245 | } |
| 246 | |
| 247 | /* This func is async.The bottom half will do the delete action at the finial |
| 248 | * end. |
| 249 | */ |
| 250 | void qemu_bh_delete(QEMUBH *bh) |
| 251 | { |
| 252 | aio_bh_enqueue(bh, BH_DELETED); |
| 253 | } |
| 254 | |
| 255 | static int64_t aio_compute_bh_timeout(BHList *head, int timeout) |
| 256 | { |
| 257 | QEMUBH *bh; |
| 258 | |
| 259 | QSLIST_FOREACH_RCU(bh, head, next) { |
| 260 | int flags = qatomic_load_acquire(&bh->flags); |
| 261 | if ((flags & (BH_SCHEDULED | BH_DELETED)) == BH_SCHEDULED) { |
| 262 | if (flags & BH_IDLE) { |
| 263 | /* idle bottom halves will be polled at least |
| 264 | * every 10ms */ |
| 265 | timeout = 10000000; |
| 266 | } else { |
| 267 | /* non-idle bottom halves will be executed |
| 268 | * immediately */ |
| 269 | return 0; |
| 270 | } |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | return timeout; |
| 275 | } |
| 276 | |
| 277 | int64_t |
| 278 | aio_compute_timeout(AioContext *ctx) |
| 279 | { |
| 280 | BHListSlice *s; |
| 281 | int64_t deadline; |
| 282 | int timeout = -1; |
| 283 | |
| 284 | timeout = aio_compute_bh_timeout(&ctx->bh_list, timeout); |
| 285 | if (timeout == 0) { |
| 286 | return 0; |
| 287 | } |
| 288 | |
| 289 | QSIMPLEQ_FOREACH(s, &ctx->bh_slice_list, next) { |
| 290 | timeout = aio_compute_bh_timeout(&s->bh_list, timeout); |
| 291 | if (timeout == 0) { |
| 292 | return 0; |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | deadline = timerlistgroup_deadline_ns(&ctx->tlg); |
| 297 | if (deadline == 0) { |
| 298 | return 0; |
| 299 | } else { |
| 300 | return qemu_soonest_timeout(timeout, deadline); |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | static gboolean |
| 305 | aio_ctx_prepare(GSource *source, gint *timeout) |
| 306 | { |
| 307 | AioContext *ctx = (AioContext *) source; |
| 308 | |
| 309 | qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) | 1); |
| 310 | |
| 311 | /* |
| 312 | * Write ctx->notify_me before computing the timeout |
| 313 | * (reading bottom half flags, etc.). Pairs with |
| 314 | * smp_mb in aio_notify(). |
| 315 | */ |
| 316 | smp_mb(); |
| 317 | |
| 318 | /* We assume there is no timeout already supplied */ |
| 319 | *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)); |
| 320 | |
| 321 | if (aio_prepare(ctx)) { |
| 322 | *timeout = 0; |
| 323 | } |
| 324 | |
| 325 | return *timeout == 0; |
| 326 | } |
| 327 | |
| 328 | static gboolean |
| 329 | aio_ctx_check(GSource *source) |
| 330 | { |
| 331 | AioContext *ctx = (AioContext *) source; |
| 332 | QEMUBH *bh; |
| 333 | BHListSlice *s; |
| 334 | |
| 335 | /* Finish computing the timeout before clearing the flag. */ |
| 336 | qatomic_store_release(&ctx->notify_me, qatomic_read(&ctx->notify_me) & ~1); |
| 337 | aio_notify_accept(ctx); |
| 338 | |
| 339 | QSLIST_FOREACH_RCU(bh, &ctx->bh_list, next) { |
| 340 | int flags = qatomic_load_acquire(&bh->flags); |
| 341 | if ((flags & (BH_SCHEDULED | BH_DELETED)) == BH_SCHEDULED) { |
| 342 | return true; |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | QSIMPLEQ_FOREACH(s, &ctx->bh_slice_list, next) { |
| 347 | QSLIST_FOREACH_RCU(bh, &s->bh_list, next) { |
| 348 | int flags = qatomic_load_acquire(&bh->flags); |
| 349 | if ((flags & (BH_SCHEDULED | BH_DELETED)) == BH_SCHEDULED) { |
| 350 | return true; |
| 351 | } |
| 352 | } |
| 353 | } |
| 354 | return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0); |
| 355 | } |
| 356 | |
| 357 | static gboolean |
| 358 | aio_ctx_dispatch(GSource *source, |
| 359 | GSourceFunc callback, |
| 360 | gpointer user_data) |
| 361 | { |
| 362 | AioContext *ctx = (AioContext *) source; |
| 363 | |
| 364 | assert(callback == NULL); |
| 365 | aio_dispatch(ctx); |
| 366 | return true; |
| 367 | } |
| 368 | |
| 369 | static void |
| 370 | aio_ctx_finalize(GSource *source) |
| 371 | { |
| 372 | AioContext *ctx = (AioContext *) source; |
| 373 | QEMUBH *bh; |
| 374 | unsigned flags; |
| 375 | |
| 376 | if (!ctx->initialized) { |
| 377 | return; |
| 378 | } |
| 379 | |
| 380 | thread_pool_free_aio(ctx->thread_pool); |
| 381 | |
| 382 | #ifdef CONFIG_LINUX_AIO |
| 383 | if (ctx->linux_aio) { |
| 384 | laio_detach_aio_context(ctx->linux_aio, ctx); |
| 385 | laio_cleanup(ctx->linux_aio); |
| 386 | ctx->linux_aio = NULL; |
| 387 | } |
| 388 | #endif |
| 389 | |
| 390 | assert(QSLIST_EMPTY(&ctx->scheduled_coroutines)); |
| 391 | qemu_bh_delete(ctx->co_schedule_bh); |
| 392 | |
| 393 | /* There must be no aio_bh_poll() calls going on */ |
| 394 | assert(QSIMPLEQ_EMPTY(&ctx->bh_slice_list)); |
| 395 | |
| 396 | while ((bh = aio_bh_dequeue(&ctx->bh_list, &flags))) { |
| 397 | /* |
| 398 | * qemu_bh_delete() must have been called on BHs in this AioContext. In |
| 399 | * many cases memory leaks, hangs, or inconsistent state occur when a |
| 400 | * BH is leaked because something still expects it to run. |
| 401 | * |
| 402 | * If you hit this, fix the lifecycle of the BH so that |
| 403 | * qemu_bh_delete() and any associated cleanup is called before the |
| 404 | * AioContext is finalized. |
| 405 | */ |
| 406 | if (unlikely(!(flags & BH_DELETED))) { |
| 407 | fprintf(stderr, "%s: BH '%s' leaked, aborting...\n", |
| 408 | __func__, bh->name); |
| 409 | abort(); |
| 410 | } |
| 411 | |
| 412 | g_free(bh); |
| 413 | } |
| 414 | |
| 415 | aio_set_event_notifier(ctx, &ctx->notifier, NULL, NULL, NULL); |
| 416 | event_notifier_cleanup(&ctx->notifier); |
| 417 | qemu_rec_mutex_destroy(&ctx->lock); |
| 418 | timerlistgroup_deinit(&ctx->tlg); |
| 419 | unregister_aiocontext(ctx); |
| 420 | aio_context_destroy(ctx); |
| 421 | /* aio_context_destroy() still needs the lock */ |
| 422 | qemu_lockcnt_destroy(&ctx->list_lock); |
| 423 | } |
| 424 | |
| 425 | static GSourceFuncs aio_source_funcs = { |
| 426 | aio_ctx_prepare, |
| 427 | aio_ctx_check, |
| 428 | aio_ctx_dispatch, |
| 429 | aio_ctx_finalize |
| 430 | }; |
| 431 | |
| 432 | GSource *aio_get_g_source(AioContext *ctx) |
| 433 | { |
| 434 | g_source_ref(&ctx->source); |
| 435 | return &ctx->source; |
| 436 | } |
| 437 | |
| 438 | ThreadPoolAio *aio_get_thread_pool(AioContext *ctx) |
| 439 | { |
| 440 | if (!ctx->thread_pool) { |
| 441 | ctx->thread_pool = thread_pool_new_aio(ctx); |
| 442 | } |
| 443 | return ctx->thread_pool; |
| 444 | } |
| 445 | |
| 446 | #ifdef CONFIG_LINUX_AIO |
| 447 | LinuxAioState *aio_setup_linux_aio(AioContext *ctx, Error **errp) |
| 448 | { |
| 449 | if (!ctx->linux_aio) { |
| 450 | ctx->linux_aio = laio_init(errp); |
| 451 | if (ctx->linux_aio) { |
| 452 | laio_attach_aio_context(ctx->linux_aio, ctx); |
| 453 | } |
| 454 | } |
| 455 | return ctx->linux_aio; |
| 456 | } |
| 457 | |
| 458 | LinuxAioState *aio_get_linux_aio(AioContext *ctx) |
| 459 | { |
| 460 | assert(ctx->linux_aio); |
| 461 | return ctx->linux_aio; |
| 462 | } |
| 463 | #endif |
| 464 | |
| 465 | void aio_notify(AioContext *ctx) |
| 466 | { |
| 467 | /* |
| 468 | * Write e.g. ctx->bh_list before writing ctx->notified. Pairs with |
| 469 | * smp_mb() in aio_notify_accept(). |
| 470 | */ |
| 471 | smp_wmb(); |
| 472 | qatomic_set(&ctx->notified, true); |
| 473 | |
| 474 | /* |
| 475 | * Write ctx->notified (and also ctx->bh_list) before reading ctx->notify_me. |
| 476 | * Pairs with smp_mb() in aio_ctx_prepare or aio_poll. |
| 477 | */ |
| 478 | smp_mb(); |
| 479 | if (qatomic_read(&ctx->notify_me)) { |
| 480 | event_notifier_set(&ctx->notifier); |
| 481 | } |
| 482 | } |
| 483 | |
| 484 | void aio_notify_accept(AioContext *ctx) |
| 485 | { |
| 486 | qatomic_set(&ctx->notified, false); |
| 487 | |
| 488 | /* |
| 489 | * Order reads of ctx->notified (in aio_context_notifier_poll()) and the |
| 490 | * above clearing of ctx->notified before reads of e.g. bh->flags. Pairs |
| 491 | * with smp_wmb() in aio_notify. |
| 492 | */ |
| 493 | smp_mb(); |
| 494 | } |
| 495 | |
| 496 | static void aio_timerlist_notify(void *opaque, QEMUClockType type) |
| 497 | { |
| 498 | aio_notify(opaque); |
| 499 | } |
| 500 | |
| 501 | static void aio_context_notifier_cb(EventNotifier *e) |
| 502 | { |
| 503 | AioContext *ctx = container_of(e, AioContext, notifier); |
| 504 | |
| 505 | event_notifier_test_and_clear(&ctx->notifier); |
| 506 | } |
| 507 | |
| 508 | /* Returns true if aio_notify() was called (e.g. a BH was scheduled) */ |
| 509 | static bool aio_context_notifier_poll(void *opaque) |
| 510 | { |
| 511 | EventNotifier *e = opaque; |
| 512 | AioContext *ctx = container_of(e, AioContext, notifier); |
| 513 | |
| 514 | /* |
| 515 | * No need for load-acquire because we just want to kick the |
| 516 | * event loop. aio_notify_accept() takes care of synchronizing |
| 517 | * the event loop with the producers. |
| 518 | */ |
| 519 | return qatomic_read(&ctx->notified); |
| 520 | } |
| 521 | |
| 522 | static void aio_context_notifier_poll_ready(EventNotifier *e) |
| 523 | { |
| 524 | /* Do nothing, we just wanted to kick the event loop */ |
| 525 | } |
| 526 | |
| 527 | static void co_schedule_bh_cb(void *opaque) |
| 528 | { |
| 529 | AioContext *ctx = opaque; |
| 530 | QSLIST_HEAD(, Coroutine) straight, reversed; |
| 531 | |
| 532 | QSLIST_MOVE_ATOMIC(&reversed, &ctx->scheduled_coroutines); |
| 533 | QSLIST_INIT(&straight); |
| 534 | |
| 535 | while (!QSLIST_EMPTY(&reversed)) { |
| 536 | Coroutine *co = QSLIST_FIRST(&reversed); |
| 537 | QSLIST_REMOVE_HEAD(&reversed, co_scheduled_next); |
| 538 | QSLIST_INSERT_HEAD(&straight, co, co_scheduled_next); |
| 539 | } |
| 540 | |
| 541 | while (!QSLIST_EMPTY(&straight)) { |
| 542 | Coroutine *co = QSLIST_FIRST(&straight); |
| 543 | QSLIST_REMOVE_HEAD(&straight, co_scheduled_next); |
| 544 | trace_aio_co_schedule_bh_cb(ctx, co); |
| 545 | |
| 546 | /* Protected by write barrier in qemu_aio_coroutine_enter */ |
| 547 | qatomic_set(&co->scheduled, NULL); |
| 548 | qemu_aio_coroutine_enter(ctx, co); |
| 549 | } |
| 550 | } |
| 551 | |
| 552 | AioContext *aio_context_new(Error **errp) |
| 553 | { |
| 554 | ERRP_GUARD(); |
| 555 | int ret; |
| 556 | AioContext *ctx; |
| 557 | |
| 558 | /* |
| 559 | * ctx is freed by g_source_unref() (e.g. aio_context_unref()). ctx's |
| 560 | * resources are freed as follows: |
| 561 | * |
| 562 | * 1. By aio_ctx_finalize() after aio_context_new() has returned and set |
| 563 | * ->initialized = true. |
| 564 | * |
| 565 | * 2. By manual cleanup code in this function's error paths before goto |
| 566 | * fail. |
| 567 | * |
| 568 | * Be careful to free resources in both cases! |
| 569 | */ |
| 570 | ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext)); |
| 571 | QSLIST_INIT(&ctx->bh_list); |
| 572 | QSIMPLEQ_INIT(&ctx->bh_slice_list); |
| 573 | |
| 574 | ret = event_notifier_init(&ctx->notifier, false); |
| 575 | if (ret < 0) { |
| 576 | error_setg_errno(errp, -ret, "Failed to initialize event notifier"); |
| 577 | goto fail; |
| 578 | } |
| 579 | |
| 580 | /* |
| 581 | * Resources cannot easily be freed manually after aio_context_setup(). If |
| 582 | * you add any new resources to AioContext, it's probably best to acquire |
| 583 | * them before aio_context_setup(). |
| 584 | */ |
| 585 | if (!aio_context_setup(ctx, errp)) { |
| 586 | event_notifier_cleanup(&ctx->notifier); |
| 587 | goto fail; |
| 588 | } |
| 589 | |
| 590 | g_source_set_can_recurse(&ctx->source, true); |
| 591 | qemu_lockcnt_init(&ctx->list_lock); |
| 592 | |
| 593 | ctx->co_schedule_bh = aio_bh_new(ctx, co_schedule_bh_cb, ctx); |
| 594 | QSLIST_INIT(&ctx->scheduled_coroutines); |
| 595 | |
| 596 | aio_set_event_notifier(ctx, &ctx->notifier, |
| 597 | aio_context_notifier_cb, |
| 598 | aio_context_notifier_poll, |
| 599 | aio_context_notifier_poll_ready); |
| 600 | #ifdef CONFIG_LINUX_AIO |
| 601 | ctx->linux_aio = NULL; |
| 602 | #endif |
| 603 | |
| 604 | ctx->thread_pool = NULL; |
| 605 | qemu_rec_mutex_init(&ctx->lock); |
| 606 | timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx); |
| 607 | |
| 608 | ctx->poll_max_ns = 0; |
| 609 | ctx->poll_ns = 0; |
| 610 | ctx->poll_grow = 0; |
| 611 | ctx->poll_shrink = 0; |
| 612 | ctx->poll_weight = 0; |
| 613 | |
| 614 | ctx->aio_max_batch = 0; |
| 615 | |
| 616 | ctx->thread_pool_min = 0; |
| 617 | ctx->thread_pool_max = THREAD_POOL_MAX_THREADS_DEFAULT; |
| 618 | |
| 619 | register_aiocontext(ctx); |
| 620 | |
| 621 | ctx->initialized = true; |
| 622 | |
| 623 | return ctx; |
| 624 | fail: |
| 625 | g_source_unref(&ctx->source); |
| 626 | return NULL; |
| 627 | } |
| 628 | |
| 629 | void aio_co_schedule(AioContext *ctx, Coroutine *co) |
| 630 | { |
| 631 | trace_aio_co_schedule(ctx, co); |
| 632 | const char *scheduled = qatomic_cmpxchg(&co->scheduled, NULL, |
| 633 | __func__); |
| 634 | |
| 635 | if (scheduled) { |
| 636 | fprintf(stderr, |
| 637 | "%s: Co-routine was already scheduled in '%s'\n", |
| 638 | __func__, scheduled); |
| 639 | abort(); |
| 640 | } |
| 641 | |
| 642 | /* The coroutine might run and release the last ctx reference before we |
| 643 | * invoke qemu_bh_schedule(). Take a reference to keep ctx alive until |
| 644 | * we're done. |
| 645 | */ |
| 646 | aio_context_ref(ctx); |
| 647 | |
| 648 | QSLIST_INSERT_HEAD_ATOMIC(&ctx->scheduled_coroutines, |
| 649 | co, co_scheduled_next); |
| 650 | qemu_bh_schedule(ctx->co_schedule_bh); |
| 651 | |
| 652 | aio_context_unref(ctx); |
| 653 | } |
| 654 | |
| 655 | typedef struct AioCoRescheduleSelf { |
| 656 | Coroutine *co; |
| 657 | AioContext *new_ctx; |
| 658 | } AioCoRescheduleSelf; |
| 659 | |
| 660 | static void aio_co_reschedule_self_bh(void *opaque) |
| 661 | { |
| 662 | AioCoRescheduleSelf *data = opaque; |
| 663 | aio_co_schedule(data->new_ctx, data->co); |
| 664 | } |
| 665 | |
| 666 | void coroutine_fn aio_co_reschedule_self(AioContext *new_ctx) |
| 667 | { |
| 668 | AioContext *old_ctx = qemu_get_current_aio_context(); |
| 669 | |
| 670 | if (old_ctx != new_ctx) { |
| 671 | AioCoRescheduleSelf data = { |
| 672 | .co = qemu_coroutine_self(), |
| 673 | .new_ctx = new_ctx, |
| 674 | }; |
| 675 | /* |
| 676 | * We can't directly schedule the coroutine in the target context |
| 677 | * because this would be racy: The other thread could try to enter the |
| 678 | * coroutine before it has yielded in this one. |
| 679 | */ |
| 680 | aio_bh_schedule_oneshot(old_ctx, aio_co_reschedule_self_bh, &data); |
| 681 | qemu_coroutine_yield(); |
| 682 | } |
| 683 | } |
| 684 | |
| 685 | void aio_co_wake(Coroutine *co) |
| 686 | { |
| 687 | AioContext *ctx; |
| 688 | |
| 689 | /* Read coroutine before co->ctx. Matches smp_wmb in |
| 690 | * qemu_coroutine_enter. |
| 691 | */ |
| 692 | smp_read_barrier_depends(); |
| 693 | ctx = qatomic_read(&co->ctx); |
| 694 | |
| 695 | aio_co_enter(ctx, co); |
| 696 | } |
| 697 | |
| 698 | void aio_co_enter(AioContext *ctx, Coroutine *co) |
| 699 | { |
| 700 | if (ctx != qemu_get_current_aio_context()) { |
| 701 | aio_co_schedule(ctx, co); |
| 702 | return; |
| 703 | } |
| 704 | |
| 705 | if (qemu_in_coroutine()) { |
| 706 | Coroutine *self = qemu_coroutine_self(); |
| 707 | assert(self != co); |
| 708 | QSIMPLEQ_INSERT_TAIL(&self->co_queue_wakeup, co, co_queue_next); |
| 709 | } else { |
| 710 | qemu_aio_coroutine_enter(ctx, co); |
| 711 | } |
| 712 | } |
| 713 | |
| 714 | void aio_context_ref(AioContext *ctx) |
| 715 | { |
| 716 | g_source_ref(&ctx->source); |
| 717 | } |
| 718 | |
| 719 | void aio_context_unref(AioContext *ctx) |
| 720 | { |
| 721 | g_source_unref(&ctx->source); |
| 722 | } |
| 723 | |
| 724 | QEMU_DEFINE_STATIC_CO_TLS(AioContext *, my_aiocontext) |
| 725 | |
| 726 | AioContext *qemu_get_current_aio_context(void) |
| 727 | { |
| 728 | AioContext *ctx = get_my_aiocontext(); |
| 729 | if (ctx) { |
| 730 | return ctx; |
| 731 | } |
| 732 | if (bql_locked()) { |
| 733 | /* Possibly in a vCPU thread. */ |
| 734 | return qemu_get_aio_context(); |
| 735 | } |
| 736 | return NULL; |
| 737 | } |
| 738 | |
| 739 | void qemu_set_current_aio_context(AioContext *ctx) |
| 740 | { |
| 741 | assert(!get_my_aiocontext()); |
| 742 | set_my_aiocontext(ctx); |
| 743 | } |
| 744 | |
| 745 | void aio_context_set_thread_pool_params(AioContext *ctx, int64_t min, |
| 746 | int64_t max, Error **errp) |
| 747 | { |
| 748 | |
| 749 | if (min > max || max <= 0 || min < 0 || min > INT_MAX || max > INT_MAX) { |
| 750 | error_setg(errp, "bad thread-pool-min/thread-pool-max values"); |
| 751 | return; |
| 752 | } |
| 753 | |
| 754 | ctx->thread_pool_min = min; |
| 755 | ctx->thread_pool_max = max; |
| 756 | |
| 757 | if (ctx->thread_pool) { |
| 758 | thread_pool_update_params(ctx->thread_pool, ctx); |
| 759 | } |
| 760 | } |