master
c 602 lines 16.7 KB
Raw
1 /*
2 * Linux native AIO support.
3 *
4 * Copyright (C) 2009 IBM, Corp.
5 * Copyright (C) 2009 Red Hat, Inc.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 */
10 #include "qemu/osdep.h"
11 #include "qemu/aio.h"
12 #include "qemu/queue.h"
13 #include "block/block.h"
14 #include "block/raw-aio.h"
15 #include "qemu/event_notifier.h"
16 #include "qemu/coroutine.h"
17 #include "qemu/defer-call.h"
18 #include "qapi/error.h"
19 #include "system/block-backend.h"
20
21 /* Only used for assertions. */
22 #include "qemu/coroutine_int.h"
23
24 #include <libaio.h>
25
26 /*
27 * Queue size (per-device).
28 *
29 * XXX: eventually we need to communicate this to the guest and/or make it
30 * tunable by the guest. If we get more outstanding requests at a time
31 * than this we will get EAGAIN from io_submit which is communicated to
32 * the guest as an I/O error.
33 */
34 #define MAX_EVENTS 1024
35
36 /* Maximum number of requests in a batch. (default value) */
37 #define DEFAULT_MAX_BATCH 32
38
39 /*
40 * Bound on how deep ioq_submit() may recurse on a single LaioQueue via the
41 * ioq_submit -> qemu_laio_process_completions -> defer_call_end ->
42 * laio_deferred_fn -> ioq_submit cycle. The cycle terminates naturally
43 * when io_submit(2) returns asynchronously (O_DIRECT), but can grow
44 * without bound when submissions complete synchronously. On overflow
45 * the caller returns without submitting; the outermost
46 * qemu_laio_process_completions() has already scheduled s->completion_bh
47 * (via qemu_bh_schedule() at the top of that function), which resumes
48 * submission from the next event-loop dispatch.
49 */
50 #define IOQ_SUBMIT_MAX_DEPTH 8
51
52 struct qemu_laiocb {
53 Coroutine *co;
54 LinuxAioState *ctx;
55 struct iocb iocb;
56 ssize_t ret;
57 off_t offset;
58 size_t nbytes;
59 QEMUIOVector *qiov;
60
61 /* For handling short reads/writes */
62 size_t total_done;
63 QEMUIOVector resubmit_qiov;
64
65 int fd;
66 int type;
67 BdrvRequestFlags flags;
68
69 uint64_t dev_max_batch;
70 QSIMPLEQ_ENTRY(qemu_laiocb) next;
71 };
72
73 typedef struct {
74 unsigned int in_queue;
75 unsigned int in_flight;
76 bool blocked;
77 unsigned int submit_depth;
78 QSIMPLEQ_HEAD(, qemu_laiocb) pending;
79 } LaioQueue;
80
81 struct LinuxAioState {
82 AioContext *aio_context;
83
84 io_context_t ctx;
85 EventNotifier e;
86
87 /* No locking required, only accessed from AioContext home thread */
88 LaioQueue io_q;
89 QEMUBH *completion_bh;
90 int event_idx;
91 int event_max;
92 };
93
94 static void ioq_submit(LinuxAioState *s);
95 static int laio_do_submit(struct qemu_laiocb *laiocb);
96
97 static inline ssize_t io_event_ret(struct io_event *ev)
98 {
99 return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res);
100 }
101
102 /**
103 * Retry tail of short requests.
104 */
105 static int laio_resubmit_short_io(struct qemu_laiocb *laiocb, size_t done)
106 {
107 QEMUIOVector *resubmit_qiov = &laiocb->resubmit_qiov;
108
109 laiocb->total_done += done;
110
111 if (!resubmit_qiov->iov) {
112 qemu_iovec_init(resubmit_qiov, laiocb->qiov->niov);
113 } else {
114 qemu_iovec_reset(resubmit_qiov);
115 }
116 qemu_iovec_concat(resubmit_qiov, laiocb->qiov,
117 laiocb->total_done, laiocb->nbytes - laiocb->total_done);
118
119 return laio_do_submit(laiocb);
120 }
121
122 /*
123 * Completes an AIO request.
124 */
125 static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)
126 {
127 ssize_t ret;
128
129 ret = laiocb->ret;
130 if (ret != -ECANCELED) {
131 if (ret == laiocb->nbytes - laiocb->total_done) {
132 ret = 0;
133 } else if (ret > 0 && (laiocb->type == QEMU_AIO_READ ||
134 laiocb->type == QEMU_AIO_WRITE)) {
135 ret = laio_resubmit_short_io(laiocb, ret);
136 if (!ret) {
137 return;
138 }
139 } else if (ret >= 0) {
140 /*
141 * For normal reads and writes, we only get here if ret == 0, which
142 * means EOF for reads and ENOSPC for writes.
143 * For zone-append, we get here with any ret >= 0, which we just
144 * treat as ENOSPC, too (safer than resubmitting, probably, but not
145 * 100 % clear).
146 */
147 if (laiocb->type == QEMU_AIO_READ) {
148 qemu_iovec_memset(laiocb->qiov, laiocb->total_done, 0,
149 laiocb->qiov->size - laiocb->total_done);
150 } else {
151 ret = -ENOSPC;
152 }
153 }
154 }
155
156 laiocb->ret = ret;
157 if (laiocb->resubmit_qiov.iov) {
158 qemu_iovec_destroy(&laiocb->resubmit_qiov);
159 }
160
161 /*
162 * If the coroutine is already entered it must be in ioq_submit() and
163 * will notice laio->ret has been filled in when it eventually runs
164 * later. Coroutines cannot be entered recursively so avoid doing
165 * that!
166 */
167 assert(laiocb->co->ctx == laiocb->ctx->aio_context);
168 if (!qemu_coroutine_entered(laiocb->co)) {
169 aio_co_wake(laiocb->co);
170 }
171 }
172
173 /**
174 * aio_ring buffer which is shared between userspace and kernel.
175 *
176 * This copied from linux/fs/aio.c, common header does not exist
177 * but AIO exists for ages so we assume ABI is stable.
178 */
179 struct aio_ring {
180 unsigned id; /* kernel internal index number */
181 unsigned nr; /* number of io_events */
182 unsigned head; /* Written to by userland or by kernel. */
183 unsigned tail;
184
185 unsigned magic;
186 unsigned compat_features;
187 unsigned incompat_features;
188 unsigned header_length; /* size of aio_ring */
189
190 struct io_event io_events[];
191 };
192
193 /**
194 * io_getevents_peek:
195 * @ctx: AIO context
196 * @events: pointer on events array, output value
197
198 * Returns the number of completed events and sets a pointer
199 * on events array. This function does not update the internal
200 * ring buffer, only reads head and tail. When @events has been
201 * processed io_getevents_commit() must be called.
202 */
203 static inline unsigned int io_getevents_peek(io_context_t ctx,
204 struct io_event **events)
205 {
206 struct aio_ring *ring = (struct aio_ring *)ctx;
207 unsigned int head = ring->head, tail = ring->tail;
208 unsigned int nr;
209
210 nr = tail >= head ? tail - head : ring->nr - head;
211 *events = ring->io_events + head;
212 /* To avoid speculative loads of s->events[i] before observing tail.
213 Paired with smp_wmb() inside linux/fs/aio.c: aio_complete(). */
214 smp_rmb();
215
216 return nr;
217 }
218
219 /**
220 * io_getevents_commit:
221 * @ctx: AIO context
222 * @nr: the number of events on which head should be advanced
223 *
224 * Advances head of a ring buffer.
225 */
226 static inline void io_getevents_commit(io_context_t ctx, unsigned int nr)
227 {
228 struct aio_ring *ring = (struct aio_ring *)ctx;
229
230 if (nr) {
231 ring->head = (ring->head + nr) % ring->nr;
232 }
233 }
234
235 /**
236 * io_getevents_advance_and_peek:
237 * @ctx: AIO context
238 * @events: pointer on events array, output value
239 * @nr: the number of events on which head should be advanced
240 *
241 * Advances head of a ring buffer and returns number of elements left.
242 */
243 static inline unsigned int
244 io_getevents_advance_and_peek(io_context_t ctx,
245 struct io_event **events,
246 unsigned int nr)
247 {
248 io_getevents_commit(ctx, nr);
249 return io_getevents_peek(ctx, events);
250 }
251
252 /**
253 * qemu_laio_process_completions:
254 * @s: AIO state
255 *
256 * Fetches completed I/O requests and invokes their callbacks.
257 *
258 * The function is somewhat tricky because it supports nested event loops, for
259 * example when a request callback invokes aio_poll(). In order to do this,
260 * indices are kept in LinuxAioState. Function schedules BH completion so it
261 * can be called again in a nested event loop. When there are no events left
262 * to complete the BH is being canceled.
263 */
264 static void qemu_laio_process_completions(LinuxAioState *s)
265 {
266 struct io_event *events;
267
268 defer_call_begin();
269
270 /* Reschedule so nested event loops see currently pending completions */
271 qemu_bh_schedule(s->completion_bh);
272
273 while ((s->event_max = io_getevents_advance_and_peek(s->ctx, &events,
274 s->event_idx))) {
275 for (s->event_idx = 0; s->event_idx < s->event_max; ) {
276 struct iocb *iocb = events[s->event_idx].obj;
277 struct qemu_laiocb *laiocb =
278 container_of(iocb, struct qemu_laiocb, iocb);
279
280 laiocb->ret = io_event_ret(&events[s->event_idx]);
281
282 /* Change counters one-by-one because we can be nested. */
283 s->io_q.in_flight--;
284 s->event_idx++;
285 qemu_laio_process_completion(laiocb);
286 }
287 }
288
289 qemu_bh_cancel(s->completion_bh);
290
291 /* If we are nested we have to notify the level above that we are done
292 * by setting event_max to zero, upper level will then jump out of it's
293 * own `for` loop. If we are the last all counters dropped to zero. */
294 s->event_max = 0;
295 s->event_idx = 0;
296
297 defer_call_end();
298 }
299
300 static void qemu_laio_process_completions_and_submit(LinuxAioState *s)
301 {
302 qemu_laio_process_completions(s);
303
304 if (!QSIMPLEQ_EMPTY(&s->io_q.pending)) {
305 ioq_submit(s);
306 }
307 }
308
309 static void qemu_laio_completion_bh(void *opaque)
310 {
311 LinuxAioState *s = opaque;
312
313 qemu_laio_process_completions_and_submit(s);
314 }
315
316 static void qemu_laio_completion_cb(EventNotifier *e)
317 {
318 LinuxAioState *s = container_of(e, LinuxAioState, e);
319
320 if (event_notifier_test_and_clear(&s->e)) {
321 qemu_laio_process_completions_and_submit(s);
322 }
323 }
324
325 static bool qemu_laio_poll_cb(void *opaque)
326 {
327 EventNotifier *e = opaque;
328 LinuxAioState *s = container_of(e, LinuxAioState, e);
329 struct io_event *events;
330
331 return io_getevents_peek(s->ctx, &events);
332 }
333
334 static void qemu_laio_poll_ready(EventNotifier *opaque)
335 {
336 EventNotifier *e = opaque;
337 LinuxAioState *s = container_of(e, LinuxAioState, e);
338
339 qemu_laio_process_completions_and_submit(s);
340 }
341
342 static void ioq_init(LaioQueue *io_q)
343 {
344 QSIMPLEQ_INIT(&io_q->pending);
345 io_q->in_queue = 0;
346 io_q->in_flight = 0;
347 io_q->blocked = false;
348 io_q->submit_depth = 0;
349 }
350
351 static void ioq_submit(LinuxAioState *s)
352 {
353 int ret, len;
354 struct qemu_laiocb *aiocb;
355 QEMU_UNINITIALIZED struct iocb *iocbs[MAX_EVENTS];
356 QSIMPLEQ_HEAD(, qemu_laiocb) completed;
357
358 if (s->io_q.submit_depth >= IOQ_SUBMIT_MAX_DEPTH) {
359 return;
360 }
361 s->io_q.submit_depth++;
362
363 do {
364 if (s->io_q.in_flight >= MAX_EVENTS) {
365 break;
366 }
367 len = 0;
368 QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) {
369 iocbs[len++] = &aiocb->iocb;
370 if (s->io_q.in_flight + len >= MAX_EVENTS) {
371 break;
372 }
373 }
374
375 ret = io_submit(s->ctx, len, iocbs);
376 if (ret == -EAGAIN) {
377 break;
378 }
379 if (ret < 0) {
380 /* Fail the first request, retry the rest */
381 aiocb = QSIMPLEQ_FIRST(&s->io_q.pending);
382 QSIMPLEQ_REMOVE_HEAD(&s->io_q.pending, next);
383 s->io_q.in_queue--;
384 aiocb->ret = ret;
385 qemu_laio_process_completion(aiocb);
386 continue;
387 }
388
389 s->io_q.in_flight += ret;
390 s->io_q.in_queue -= ret;
391 aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb);
392 QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed);
393 } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending));
394 s->io_q.blocked = (s->io_q.in_queue > 0);
395
396 if (s->io_q.in_flight) {
397 /* We can try to complete something just right away if there are
398 * still requests in-flight. */
399 qemu_laio_process_completions(s);
400 /*
401 * Even we have completed everything (in_flight == 0), the queue can
402 * have still pended requests (in_queue > 0). We do not attempt to
403 * repeat submission to avoid IO hang. The reason is simple: s->e is
404 * still set and completion callback will be called shortly and all
405 * pended requests will be submitted from there.
406 */
407 }
408
409 s->io_q.submit_depth--;
410 }
411
412 static uint64_t laio_max_batch(LinuxAioState *s, uint64_t dev_max_batch)
413 {
414 uint64_t max_batch = s->aio_context->aio_max_batch ?: DEFAULT_MAX_BATCH;
415
416 /*
417 * AIO context can be shared between multiple block devices, so
418 * `dev_max_batch` allows reducing the batch size for latency-sensitive
419 * devices.
420 */
421 max_batch = MIN_NON_ZERO(dev_max_batch, max_batch);
422
423 /* limit the batch with the number of available events */
424 max_batch = MIN_NON_ZERO(MAX_EVENTS - s->io_q.in_flight, max_batch);
425
426 return max_batch;
427 }
428
429 static void laio_deferred_fn(void *opaque)
430 {
431 LinuxAioState *s = opaque;
432
433 if (!s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
434 ioq_submit(s);
435 }
436 }
437
438 static int laio_do_submit(struct qemu_laiocb *laiocb)
439 {
440 LinuxAioState *s = laiocb->ctx;
441 struct iocb *iocbs = &laiocb->iocb;
442 QEMUIOVector *qiov = laiocb->qiov;
443 int fd = laiocb->fd;
444 off_t offset = laiocb->offset + laiocb->total_done;
445
446 if (laiocb->resubmit_qiov.iov) {
447 qiov = &laiocb->resubmit_qiov;
448 }
449
450 switch (laiocb->type) {
451 case QEMU_AIO_WRITE:
452 #ifdef HAVE_IO_PREP_PWRITEV2
453 {
454 int laio_flags = (laiocb->flags & BDRV_REQ_FUA) ? RWF_DSYNC : 0;
455 io_prep_pwritev2(iocbs, fd, qiov->iov, qiov->niov, offset, laio_flags);
456 }
457 #else
458 assert(laiocb->flags == 0);
459 io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset);
460 #endif
461 break;
462 case QEMU_AIO_ZONE_APPEND:
463 io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset);
464 break;
465 case QEMU_AIO_READ:
466 io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset);
467 break;
468 case QEMU_AIO_FLUSH:
469 io_prep_fdsync(iocbs, fd);
470 break;
471 /* Currently Linux kernel does not support other operations */
472 default:
473 fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
474 __func__, laiocb->type);
475 return -EIO;
476 }
477 io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
478
479 QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next);
480 s->io_q.in_queue++;
481 if (!s->io_q.blocked) {
482 if (s->io_q.in_queue >= laio_max_batch(s, laiocb->dev_max_batch)) {
483 ioq_submit(s);
484 } else {
485 defer_call(laio_deferred_fn, s);
486 }
487 }
488
489 return 0;
490 }
491
492 int coroutine_fn laio_co_submit(int fd, uint64_t offset, QEMUIOVector *qiov,
493 int type, BdrvRequestFlags flags,
494 uint64_t dev_max_batch)
495 {
496 int ret;
497 AioContext *ctx = qemu_get_current_aio_context();
498 struct qemu_laiocb laiocb = {
499 .co = qemu_coroutine_self(),
500 .offset = offset,
501 .nbytes = qiov ? qiov->size : 0,
502 .ctx = aio_get_linux_aio(ctx),
503 .ret = -EINPROGRESS,
504 .qiov = qiov,
505 .fd = fd,
506 .type = type,
507 .flags = flags,
508 .dev_max_batch = dev_max_batch,
509 };
510
511 ret = laio_do_submit(&laiocb);
512 if (ret < 0) {
513 return ret;
514 }
515
516 if (laiocb.ret == -EINPROGRESS) {
517 qemu_coroutine_yield();
518 }
519 return laiocb.ret;
520 }
521
522 void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context)
523 {
524 aio_set_event_notifier(old_context, &s->e, NULL, NULL, NULL);
525 qemu_bh_delete(s->completion_bh);
526 s->aio_context = NULL;
527 }
528
529 void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context)
530 {
531 s->aio_context = new_context;
532 s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
533 aio_set_event_notifier(new_context, &s->e,
534 qemu_laio_completion_cb,
535 qemu_laio_poll_cb,
536 qemu_laio_poll_ready);
537 }
538
539 LinuxAioState *laio_init(Error **errp)
540 {
541 int rc;
542 LinuxAioState *s;
543
544 s = g_malloc0(sizeof(*s));
545 rc = event_notifier_init(&s->e, false);
546 if (rc < 0) {
547 error_setg_errno(errp, -rc, "failed to initialize event notifier");
548 goto out_free_state;
549 }
550
551 rc = io_setup(MAX_EVENTS, &s->ctx);
552 if (rc < 0) {
553 error_setg_errno(errp, -rc, "failed to create linux AIO context");
554 goto out_close_efd;
555 }
556
557 ioq_init(&s->io_q);
558
559 return s;
560
561 out_close_efd:
562 event_notifier_cleanup(&s->e);
563 out_free_state:
564 g_free(s);
565 return NULL;
566 }
567
568 void laio_cleanup(LinuxAioState *s)
569 {
570 event_notifier_cleanup(&s->e);
571
572 if (io_destroy(s->ctx) != 0) {
573 fprintf(stderr, "%s: destroy AIO context %p failed\n",
574 __func__, &s->ctx);
575 }
576 g_free(s);
577 }
578
579 bool laio_has_fdsync(int fd)
580 {
581 struct iocb cb;
582 struct iocb *cbs[] = {&cb, NULL};
583
584 io_context_t ctx = 0;
585 io_setup(1, &ctx);
586
587 /* check if host kernel supports IO_CMD_FDSYNC */
588 io_prep_fdsync(&cb, fd);
589 int ret = io_submit(ctx, 1, cbs);
590
591 io_destroy(ctx);
592 return (ret == -EINVAL) ? false : true;
593 }
594
595 bool laio_has_fua(void)
596 {
597 #ifdef HAVE_IO_PREP_PWRITEV2
598 return true;
599 #else
600 return false;
601 #endif
602 }