block-backend.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,875 lines 76.7 KB

Raw

1	/*
2	* QEMU Block backends
3	*
4	* Copyright (C) 2014-2016 Red Hat, Inc.
5	*
6	* Authors:
7	* Markus Armbruster <armbru@redhat.com>,
8	*
9	* This work is licensed under the terms of the GNU LGPL, version 2.1
10	* or later. See the COPYING.LIB file in the top-level directory.
11	*/
12
13	#include "qemu/osdep.h"
14	#include "system/block-backend.h"
15	#include "block/block_int.h"
16	#include "block/blockjob.h"
17	#include "block/coroutines.h"
18	#include "block/throttle-groups.h"
19	#include "hw/core/qdev.h"
20	#include "system/blockdev.h"
21	#include "system/runstate.h"
22	#include "system/replay.h"
23	#include "qapi/error.h"
24	#include "qapi/qapi-events-block.h"
25	#include "qemu/id.h"
26	#include "qemu/main-loop.h"
27	#include "qemu/option.h"
28	#include "trace.h"
29	#include "migration/misc.h"
30
31	/* Number of coroutines to reserve per attached device model */
32	#define COROUTINE_POOL_RESERVATION 64
33
34	#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35
36	typedef struct BlockBackendAioNotifier {
37	void (attached_aio_context)(AioContext new_context, void *opaque);
38	void (detach_aio_context)(void opaque);
39	void *opaque;
40	QLIST_ENTRY(BlockBackendAioNotifier) list;
41	} BlockBackendAioNotifier;
42
43	struct BlockBackend {
44	char *name;
45	int refcnt;
46	BdrvChild *root;
47	AioContext ctx; / access with atomic operations only */
48	DriveInfo legacy_dinfo; / null unless created by drive_new() */
49	QTAILQ_ENTRY(BlockBackend) link; /* for block_backends */
50	QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */
51	BlockBackendPublic public;
52
53	DeviceState dev; / attached device model, if any */
54	const BlockDevOps *dev_ops;
55	void *dev_opaque;
56
57	/* If the BDS tree is removed, some of its options are stored here (which
58	* can be used to restore those options in the new BDS on insert) */
59	BlockBackendRootState root_state;
60
61	bool enable_write_cache;
62
63	/* I/O stats (display with "info blockstats"). */
64	BlockAcctStats stats;
65
66	BlockdevOnError on_read_error, on_write_error;
67	bool iostatus_enabled;
68	BlockDeviceIoStatus iostatus;
69
70	uint64_t perm;
71	uint64_t shared_perm;
72	bool disable_perm;
73
74	bool allow_aio_context_change;
75	bool allow_write_beyond_eof;
76
77	/* Protected by BQL */
78	NotifierList remove_bs_notifiers, insert_bs_notifiers;
79	QLIST_HEAD(, BlockBackendAioNotifier) aio_notifiers;
80
81	int quiesce_counter; /* atomic: written under BQL, read by other threads */
82	QemuMutex queued_requests_lock; /* protects queued_requests */
83	CoQueue queued_requests;
84	bool disable_request_queuing; /* atomic */
85	int start_request_count; /* atomic */
86
87	VMChangeStateEntry *vmsh;
88	bool force_allow_inactivate;
89
90	/* Number of in-flight aio requests. BlockDriverState also counts
91	* in-flight requests but aio requests can exist even when blk->root is
92	* NULL, so we cannot rely on its counter for that case.
93	* Accessed with atomic ops.
94	*/
95	unsigned int in_flight;
96	};
97
98	typedef struct BlockBackendAIOCB {
99	BlockAIOCB common;
100	BlockBackend *blk;
101	int ret;
102	} BlockBackendAIOCB;
103
104	static const AIOCBInfo block_backend_aiocb_info = {
105	.aiocb_size = sizeof(BlockBackendAIOCB),
106	};
107
108	static void drive_info_del(DriveInfo *dinfo);
109	static BlockBackend bdrv_first_blk(BlockDriverState bs);
110
111	/* All BlockBackends. Protected by BQL. */
112	static QTAILQ_HEAD(, BlockBackend) block_backends =
113	QTAILQ_HEAD_INITIALIZER(block_backends);
114
115	/*
116	* All BlockBackends referenced by the monitor and which are iterated through by
117	* blk_next(). Protected by BQL.
118	*/
119	static QTAILQ_HEAD(, BlockBackend) monitor_block_backends =
120	QTAILQ_HEAD_INITIALIZER(monitor_block_backends);
121
122	static int coroutine_mixed_fn GRAPH_RDLOCK
123	blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
124	Error **errp);
125
126	static void blk_root_inherit_options(BdrvChildRole role, bool parent_is_format,
127	int child_flags, QDict child_options,
128	int parent_flags, QDict *parent_options)
129	{
130	/* We're not supposed to call this function for root nodes */
131	abort();
132	}
133	static void blk_root_drained_begin(BdrvChild *child);
134	static bool blk_root_drained_poll(BdrvChild *child);
135	static void blk_root_drained_end(BdrvChild *child);
136
137	static void blk_root_change_media(BdrvChild *child, bool load);
138	static void blk_root_resize(BdrvChild *child);
139
140	static bool GRAPH_RDLOCK
141	blk_root_change_aio_ctx(BdrvChild child, AioContext ctx, GHashTable *visited,
142	Transaction tran, Error *errp);
143
144	static char blk_root_get_parent_desc(BdrvChild child)
145	{
146	BlockBackend *blk = child->opaque;
147	g_autofree char *dev_id = NULL;
148
149	if (blk->name) {
150	return g_strdup_printf("block device '%s'", blk->name);
151	}
152
153	dev_id = blk_get_attached_dev_id(blk);
154	if (*dev_id) {
155	return g_strdup_printf("block device '%s'", dev_id);
156	} else {
157	/* TODO Callback into the BB owner for something more detailed */
158	return g_strdup("an unnamed block device");
159	}
160	}
161
162	static const char blk_root_get_name(BdrvChild child)
163	{
164	return blk_name(child->opaque);
165	}
166
167	static void blk_vm_state_changed(void *opaque, bool running, RunState state)
168	{
169	Error *local_err = NULL;
170	BlockBackend *blk = opaque;
171
172	if (state == RUN_STATE_INMIGRATE) {
173	return;
174	}
175
176	qemu_del_vm_change_state_handler(blk->vmsh);
177	blk->vmsh = NULL;
178	blk_set_perm(blk, blk->perm, blk->shared_perm, &local_err);
179	if (local_err) {
180	error_report_err(local_err);
181	}
182	}
183
184	/*
185	* Notifies the user of the BlockBackend that migration has completed. qdev
186	* devices can tighten their permissions in response (specifically revoke
187	* shared write permissions that we needed for storage migration).
188	*
189	* If an error is returned, the VM cannot be allowed to be resumed.
190	*/
191	static void GRAPH_RDLOCK blk_root_activate(BdrvChild child, Error *errp)
192	{
193	BlockBackend *blk = child->opaque;
194	Error *local_err = NULL;
195	uint64_t saved_shared_perm;
196
197	if (!blk->disable_perm) {
198	return;
199	}
200
201	blk->disable_perm = false;
202
203	/*
204	* blk->shared_perm contains the permissions we want to share once
205	* migration is really completely done. For now, we need to share
206	* all; but we also need to retain blk->shared_perm, which is
207	* overwritten by a successful blk_set_perm() call. Save it and
208	* restore it below.
209	*/
210	saved_shared_perm = blk->shared_perm;
211
212	blk_set_perm_locked(blk, blk->perm, BLK_PERM_ALL, &local_err);
213	if (local_err) {
214	error_propagate(errp, local_err);
215	blk->disable_perm = true;
216	return;
217	}
218	blk->shared_perm = saved_shared_perm;
219
220	if (runstate_check(RUN_STATE_INMIGRATE)) {
221	/* Activation can happen when migration process is still active, for
222	* example when nbd_server_add is called during non-shared storage
223	* migration. Defer the shared_perm update to migration completion. */
224	if (!blk->vmsh) {
225	blk->vmsh = qemu_add_vm_change_state_handler(blk_vm_state_changed,
226	blk);
227	}
228	return;
229	}
230
231	blk_set_perm_locked(blk, blk->perm, blk->shared_perm, &local_err);
232	if (local_err) {
233	error_propagate(errp, local_err);
234	blk->disable_perm = true;
235	return;
236	}
237	}
238
239	void blk_set_force_allow_inactivate(BlockBackend *blk)
240	{
241	GLOBAL_STATE_CODE();
242	blk->force_allow_inactivate = true;
243	}
244
245	static bool blk_can_inactivate(BlockBackend *blk)
246	{
247	/* If it is a guest device, inactivate is ok. */
248	if (blk->dev \|\| blk_name(blk)[0]) {
249	return true;
250	}
251
252	/* Inactivating means no more writes to the image can be done,
253	* even if those writes would be changes invisible to the
254	* guest. For block job BBs that satisfy this, we can just allow
255	* it. This is the case for mirror job source, which is required
256	* by libvirt non-shared block migration. */
257	if (!(blk->perm & ~BLK_PERM_CONSISTENT_READ)) {
258	return true;
259	}
260
261	return blk->force_allow_inactivate;
262	}
263
264	static int GRAPH_RDLOCK blk_root_inactivate(BdrvChild *child)
265	{
266	BlockBackend *blk = child->opaque;
267
268	if (blk->disable_perm) {
269	return 0;
270	}
271
272	if (!blk_can_inactivate(blk)) {
273	return -EPERM;
274	}
275
276	blk->disable_perm = true;
277	if (blk->root) {
278	bdrv_child_try_set_perm(blk->root, 0, BLK_PERM_ALL, &error_abort);
279	}
280
281	return 0;
282	}
283
284	static void blk_root_attach(BdrvChild *child)
285	{
286	BlockBackend *blk = child->opaque;
287	BlockBackendAioNotifier *notifier;
288
289	trace_blk_root_attach(child, blk, child->bs);
290
291	QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
292	bdrv_add_aio_context_notifier(child->bs,
293	notifier->attached_aio_context,
294	notifier->detach_aio_context,
295	notifier->opaque);
296	}
297	}
298
299	static void blk_root_detach(BdrvChild *child)
300	{
301	BlockBackend *blk = child->opaque;
302	BlockBackendAioNotifier *notifier;
303
304	trace_blk_root_detach(child, blk, child->bs);
305
306	QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
307	bdrv_remove_aio_context_notifier(child->bs,
308	notifier->attached_aio_context,
309	notifier->detach_aio_context,
310	notifier->opaque);
311	}
312	}
313
314	static AioContext blk_root_get_parent_aio_context(BdrvChild c)
315	{
316	BlockBackend *blk = c->opaque;
317	IO_CODE();
318
319	return blk_get_aio_context(blk);
320	}
321
322	static const BdrvChildClass child_root = {
323	.inherit_options = blk_root_inherit_options,
324
325	.change_media = blk_root_change_media,
326	.resize = blk_root_resize,
327	.get_name = blk_root_get_name,
328	.get_parent_desc = blk_root_get_parent_desc,
329
330	.drained_begin = blk_root_drained_begin,
331	.drained_poll = blk_root_drained_poll,
332	.drained_end = blk_root_drained_end,
333
334	.activate = blk_root_activate,
335	.inactivate = blk_root_inactivate,
336
337	.attach = blk_root_attach,
338	.detach = blk_root_detach,
339
340	.change_aio_ctx = blk_root_change_aio_ctx,
341
342	.get_parent_aio_context = blk_root_get_parent_aio_context,
343	};
344
345	/*
346	* Create a new BlockBackend with a reference count of one.
347	*
348	* @perm is a bitmasks of BLK_PERM_* constants which describes the permissions
349	* to request for a block driver node that is attached to this BlockBackend.
350	* @shared_perm is a bitmask which describes which permissions may be granted
351	* to other users of the attached node.
352	* Both sets of permissions can be changed later using blk_set_perm().
353	*
354	* Return the new BlockBackend on success, null on failure.
355	*/
356	BlockBackend blk_new(AioContext ctx, uint64_t perm, uint64_t shared_perm)
357	{
358	BlockBackend *blk;
359
360	GLOBAL_STATE_CODE();
361
362	blk = g_new0(BlockBackend, 1);
363	blk->refcnt = 1;
364	blk->ctx = ctx;
365	blk->perm = perm;
366	blk->shared_perm = shared_perm;
367	blk_set_enable_write_cache(blk, true);
368
369	blk->on_read_error = BLOCKDEV_ON_ERROR_REPORT;
370	blk->on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
371
372	block_acct_init(&blk->stats);
373
374	qemu_mutex_init(&blk->queued_requests_lock);
375	qemu_co_queue_init(&blk->queued_requests);
376	notifier_list_init(&blk->remove_bs_notifiers);
377	notifier_list_init(&blk->insert_bs_notifiers);
378	QLIST_INIT(&blk->aio_notifiers);
379
380	QTAILQ_INSERT_TAIL(&block_backends, blk, link);
381	return blk;
382	}
383
384	/*
385	* Create a new BlockBackend connected to an existing BlockDriverState.
386	*
387	* @perm is a bitmasks of BLK_PERM_* constants which describes the
388	* permissions to request for @bs that is attached to this
389	* BlockBackend. @shared_perm is a bitmask which describes which
390	* permissions may be granted to other users of the attached node.
391	* Both sets of permissions can be changed later using blk_set_perm().
392	*
393	* Return the new BlockBackend on success, null on failure.
394	*/
395	BlockBackend blk_new_with_bs(BlockDriverState bs, uint64_t perm,
396	uint64_t shared_perm, Error **errp)
397	{
398	BlockBackend *blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
399
400	GLOBAL_STATE_CODE();
401
402	if (blk_insert_bs(blk, bs, errp) < 0) {
403	blk_unref(blk);
404	return NULL;
405	}
406	return blk;
407	}
408
409	/*
410	* Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
411	* By default, the new BlockBackend is in the main AioContext, but if the
412	* parameters connect it with any existing node in a different AioContext, it
413	* may end up there instead.
414	*
415	* Just as with bdrv_open(), after having called this function the reference to
416	* @options belongs to the block layer (even on failure).
417	*
418	* TODO: Remove @filename and @flags; it should be possible to specify a whole
419	* BDS tree just by specifying the @options QDict (or @reference,
420	* alternatively). At the time of adding this function, this is not possible,
421	* though, so callers of this function have to be able to specify @filename and
422	* @flags.
423	*/
424	BlockBackend blk_new_open(const char filename, const char *reference,
425	QDict options, int flags, Error *errp)
426	{
427	BlockBackend *blk;
428	BlockDriverState *bs;
429	uint64_t perm = 0;
430	uint64_t shared = BLK_PERM_ALL;
431
432	GLOBAL_STATE_CODE();
433
434	/*
435	* blk_new_open() is mainly used in .bdrv_create implementations and the
436	* tools where sharing isn't a major concern because the BDS stays private
437	* and the file is generally not supposed to be used by a second process,
438	* so we just request permission according to the flags.
439	*
440	* The exceptions are xen_disk and blockdev_init(); in these cases, the
441	* caller of blk_new_open() doesn't make use of the permissions, but they
442	* shouldn't hurt either. We can still share everything here because the
443	* guest devices will add their own blockers if they can't share.
444	*/
445	if ((flags & BDRV_O_NO_IO) == 0) {
446	perm \|= BLK_PERM_CONSISTENT_READ;
447	if (flags & BDRV_O_RDWR) {
448	perm \|= BLK_PERM_WRITE;
449	}
450	}
451	if (flags & BDRV_O_RESIZE) {
452	perm \|= BLK_PERM_RESIZE;
453	}
454	if (flags & BDRV_O_NO_SHARE) {
455	shared = BLK_PERM_CONSISTENT_READ \| BLK_PERM_WRITE_UNCHANGED;
456	}
457
458	bs = bdrv_open(filename, reference, options, flags, errp);
459	if (!bs) {
460	return NULL;
461	}
462
463	/* bdrv_open() could have moved bs to a different AioContext */
464	blk = blk_new(bdrv_get_aio_context(bs), perm, shared);
465	blk->perm = perm;
466	blk->shared_perm = shared;
467
468	blk_insert_bs(blk, bs, errp);
469	bdrv_unref(bs);
470
471	if (!blk->root) {
472	blk_unref(blk);
473	return NULL;
474	}
475
476	return blk;
477	}
478
479	static void blk_delete(BlockBackend *blk)
480	{
481	assert(!blk->refcnt);
482	assert(!blk->name);
483	assert(!blk->dev);
484	if (blk->public.throttle_group_member.throttle_state) {
485	blk_io_limits_disable(blk);
486	}
487	if (blk->root) {
488	blk_remove_bs(blk);
489	}
490	if (blk->vmsh) {
491	qemu_del_vm_change_state_handler(blk->vmsh);
492	blk->vmsh = NULL;
493	}
494	assert(QLIST_EMPTY(&blk->remove_bs_notifiers.notifiers));
495	assert(QLIST_EMPTY(&blk->insert_bs_notifiers.notifiers));
496	assert(QLIST_EMPTY(&blk->aio_notifiers));
497	assert(qemu_co_queue_empty(&blk->queued_requests));
498	qemu_mutex_destroy(&blk->queued_requests_lock);
499	QTAILQ_REMOVE(&block_backends, blk, link);
500	drive_info_del(blk->legacy_dinfo);
501	block_acct_cleanup(&blk->stats);
502	g_free(blk);
503	}
504
505	static void drive_info_del(DriveInfo *dinfo)
506	{
507	if (!dinfo) {
508	return;
509	}
510	qemu_opts_del(dinfo->opts);
511	g_free(dinfo);
512	}
513
514	int blk_get_refcnt(BlockBackend *blk)
515	{
516	GLOBAL_STATE_CODE();
517	return blk ? blk->refcnt : 0;
518	}
519
520	/*
521	* Increment @blk's reference count.
522	* @blk must not be null.
523	*/
524	void blk_ref(BlockBackend *blk)
525	{
526	assert(blk->refcnt > 0);
527	GLOBAL_STATE_CODE();
528	blk->refcnt++;
529	}
530
531	/*
532	* Decrement @blk's reference count.
533	* If this drops it to zero, destroy @blk.
534	* For convenience, do nothing if @blk is null.
535	*/
536	void blk_unref(BlockBackend *blk)
537	{
538	GLOBAL_STATE_CODE();
539	if (blk) {
540	assert(blk->refcnt > 0);
541	if (blk->refcnt > 1) {
542	blk->refcnt--;
543	} else {
544	blk_drain(blk);
545	/* blk_drain() cannot resurrect blk, nobody held a reference */
546	assert(blk->refcnt == 1);
547	blk->refcnt = 0;
548	blk_delete(blk);
549	}
550	}
551	}
552
553	/*
554	* Behaves similarly to blk_next() but iterates over all BlockBackends, even the
555	* ones which are hidden (i.e. are not referenced by the monitor).
556	*/
557	BlockBackend blk_all_next(BlockBackend blk)
558	{
559	GLOBAL_STATE_CODE();
560	return blk ? QTAILQ_NEXT(blk, link)
561	: QTAILQ_FIRST(&block_backends);
562	}
563
564	void blk_remove_all_bs(void)
565	{
566	BlockBackend *blk = NULL;
567
568	GLOBAL_STATE_CODE();
569
570	while ((blk = blk_all_next(blk)) != NULL) {
571	if (blk->root) {
572	blk_remove_bs(blk);
573	}
574	}
575	}
576
577	/*
578	* Return the monitor-owned BlockBackend after @blk.
579	* If @blk is null, return the first one.
580	* Else, return @blk's next sibling, which may be null.
581	*
582	* To iterate over all BlockBackends, do
583	* for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
584	* ...
585	* }
586	*/
587	BlockBackend blk_next(BlockBackend blk)
588	{
589	GLOBAL_STATE_CODE();
590	return blk ? QTAILQ_NEXT(blk, monitor_link)
591	: QTAILQ_FIRST(&monitor_block_backends);
592	}
593
594	/* Iterates over all top-level BlockDriverStates, i.e. BDSs that are owned by
595	* the monitor or attached to a BlockBackend */
596	BlockDriverState bdrv_next(BdrvNextIterator it)
597	{
598	BlockDriverState bs, old_bs;
599
600	/* Must be called from the main loop */
601	assert(qemu_get_current_aio_context() == qemu_get_aio_context());
602
603	old_bs = it->bs;
604
605	/* First, return all root nodes of BlockBackends. In order to avoid
606	* returning a BDS twice when multiple BBs refer to it, we only return it
607	* if the BB is the first one in the parent list of the BDS. */
608	if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
609	BlockBackend *old_blk = it->blk;
610
611	do {
612	it->blk = blk_all_next(it->blk);
613	bs = it->blk ? blk_bs(it->blk) : NULL;
614	} while (it->blk && (bs == NULL \|\| bdrv_first_blk(bs) != it->blk));
615
616	if (it->blk) {
617	blk_ref(it->blk);
618	}
619	blk_unref(old_blk);
620
621	if (bs) {
622	bdrv_ref(bs);
623	bdrv_unref(old_bs);
624	it->bs = bs;
625	return bs;
626	}
627	it->phase = BDRV_NEXT_MONITOR_OWNED;
628	}
629
630	/* Then return the monitor-owned BDSes without a BB attached. Ignore all
631	* BDSes that are attached to a BlockBackend here; they have been handled
632	* by the above block already */
633	do {
634	it->bs = bdrv_next_monitor_owned(it->bs);
635	bs = it->bs;
636	} while (bs && bdrv_has_blk(bs));
637
638	if (bs) {
639	bdrv_ref(bs);
640	}
641	bdrv_unref(old_bs);
642
643	return bs;
644	}
645
646	static void bdrv_next_reset(BdrvNextIterator *it)
647	{
648	*it = (BdrvNextIterator) {
649	.phase = BDRV_NEXT_BACKEND_ROOTS,
650	};
651	}
652
653	BlockDriverState bdrv_first(BdrvNextIterator it)
654	{
655	GLOBAL_STATE_CODE();
656	bdrv_next_reset(it);
657	return bdrv_next(it);
658	}
659
660	/* Must be called when aborting a bdrv_next() iteration before
661	* bdrv_next() returns NULL */
662	void bdrv_next_cleanup(BdrvNextIterator *it)
663	{
664	/* Must be called from the main loop */
665	assert(qemu_get_current_aio_context() == qemu_get_aio_context());
666
667	bdrv_unref(it->bs);
668
669	if (it->phase == BDRV_NEXT_BACKEND_ROOTS && it->blk) {
670	blk_unref(it->blk);
671	}
672
673	bdrv_next_reset(it);
674	}
675
676	/*
677	* Add a BlockBackend into the list of backends referenced by the monitor, with
678	* the given @name acting as the handle for the monitor.
679	* Strictly for use by blockdev.c.
680	*
681	* @name must not be null or empty.
682	*
683	* Returns true on success and false on failure. In the latter case, an Error
684	* object is returned through @errp.
685	*/
686	bool monitor_add_blk(BlockBackend blk, const char name, Error **errp)
687	{
688	assert(!blk->name);
689	assert(name && name[0]);
690	GLOBAL_STATE_CODE();
691
692	if (!id_wellformed(name)) {
693	error_setg(errp, "Invalid device name");
694	return false;
695	}
696	if (blk_by_name(name)) {
697	error_setg(errp, "Device with id '%s' already exists", name);
698	return false;
699	}
700	if (bdrv_find_node(name)) {
701	error_setg(errp,
702	"Device name '%s' conflicts with an existing node name",
703	name);
704	return false;
705	}
706
707	blk->name = g_strdup(name);
708	QTAILQ_INSERT_TAIL(&monitor_block_backends, blk, monitor_link);
709	return true;
710	}
711
712	/*
713	* Remove a BlockBackend from the list of backends referenced by the monitor.
714	* Strictly for use by blockdev.c.
715	*/
716	void monitor_remove_blk(BlockBackend *blk)
717	{
718	GLOBAL_STATE_CODE();
719
720	if (!blk->name) {
721	return;
722	}
723
724	QTAILQ_REMOVE(&monitor_block_backends, blk, monitor_link);
725	g_free(blk->name);
726	blk->name = NULL;
727	}
728
729	/*
730	* Return @blk's name, a non-null string.
731	* Returns an empty string iff @blk is not referenced by the monitor.
732	*/
733	const char blk_name(const BlockBackend blk)
734	{
735	IO_CODE();
736	return blk->name ?: "";
737	}
738
739	/*
740	* Return the BlockBackend with name @name if it exists, else null.
741	* @name must not be null.
742	*/
743	BlockBackend blk_by_name(const char name)
744	{
745	BlockBackend *blk = NULL;
746
747	GLOBAL_STATE_CODE();
748	assert(name);
749	while ((blk = blk_next(blk)) != NULL) {
750	if (!strcmp(name, blk->name)) {
751	return blk;
752	}
753	}
754	return NULL;
755	}
756
757	/*
758	* Return the BlockDriverState attached to @blk if any, else null.
759	*/
760	BlockDriverState blk_bs(BlockBackend blk)
761	{
762	IO_CODE();
763	return blk->root ? blk->root->bs : NULL;
764	}
765
766	static BlockBackend * GRAPH_RDLOCK bdrv_first_blk(BlockDriverState *bs)
767	{
768	BdrvChild *child;
769
770	GLOBAL_STATE_CODE();
771	assert_bdrv_graph_readable();
772
773	QLIST_FOREACH(child, &bs->parents, next_parent) {
774	if (child->klass == &child_root) {
775	return child->opaque;
776	}
777	}
778
779	return NULL;
780	}
781
782	/*
783	* Returns true if @bs has an associated BlockBackend.
784	*/
785	bool bdrv_has_blk(BlockDriverState *bs)
786	{
787	GLOBAL_STATE_CODE();
788	return bdrv_first_blk(bs) != NULL;
789	}
790
791	/*
792	* Returns true if @bs has only BlockBackends as parents.
793	*/
794	bool bdrv_is_root_node(BlockDriverState *bs)
795	{
796	BdrvChild *c;
797
798	GLOBAL_STATE_CODE();
799	assert_bdrv_graph_readable();
800
801	QLIST_FOREACH(c, &bs->parents, next_parent) {
802	if (c->klass != &child_root) {
803	return false;
804	}
805	}
806
807	return true;
808	}
809
810	/*
811	* Return @blk's DriveInfo if any, else null.
812	*/
813	DriveInfo blk_legacy_dinfo(BlockBackend blk)
814	{
815	GLOBAL_STATE_CODE();
816	return blk->legacy_dinfo;
817	}
818
819	/*
820	* Set @blk's DriveInfo to @dinfo, and return it.
821	* @blk must not have a DriveInfo set already.
822	* No other BlockBackend may have the same DriveInfo set.
823	*/
824	DriveInfo blk_set_legacy_dinfo(BlockBackend blk, DriveInfo *dinfo)
825	{
826	assert(!blk->legacy_dinfo);
827	GLOBAL_STATE_CODE();
828	return blk->legacy_dinfo = dinfo;
829	}
830
831	/*
832	* Return the BlockBackend with DriveInfo @dinfo.
833	* It must exist.
834	*/
835	BlockBackend blk_by_legacy_dinfo(DriveInfo dinfo)
836	{
837	BlockBackend *blk = NULL;
838	GLOBAL_STATE_CODE();
839
840	while ((blk = blk_next(blk)) != NULL) {
841	if (blk->legacy_dinfo == dinfo) {
842	return blk;
843	}
844	}
845	abort();
846	}
847
848	/*
849	* Returns a pointer to the publicly accessible fields of @blk.
850	*/
851	BlockBackendPublic blk_get_public(BlockBackend blk)
852	{
853	GLOBAL_STATE_CODE();
854	return &blk->public;
855	}
856
857	/*
858	* Disassociates the currently associated BlockDriverState from @blk.
859	*/
860	void blk_remove_bs(BlockBackend *blk)
861	{
862	ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
863	BdrvChild *root;
864
865	GLOBAL_STATE_CODE();
866
867	notifier_list_notify(&blk->remove_bs_notifiers, blk);
868	if (tgm->throttle_state) {
869	BlockDriverState *bs = blk_bs(blk);
870
871	/*
872	* Take a ref in case blk_bs() changes across bdrv_drained_begin(), for
873	* example, if a temporary filter node is removed by a blockjob.
874	*/
875	bdrv_ref(bs);
876	bdrv_drained_begin(bs);
877	throttle_group_detach_aio_context(tgm);
878	throttle_group_attach_aio_context(tgm, qemu_get_aio_context());
879	bdrv_drained_end(bs);
880	bdrv_unref(bs);
881	}
882
883	blk_update_root_state(blk);
884
885	/* bdrv_root_unref_child() will cause blk->root to become stale and may
886	* switch to a completion coroutine later on. Let's drain all I/O here
887	* to avoid that and a potential QEMU crash.
888	*/
889	blk_drain(blk);
890	root = blk->root;
891	blk->root = NULL;
892
893	bdrv_graph_wrlock_drained();
894	bdrv_root_unref_child(root);
895	bdrv_graph_wrunlock();
896	}
897
898	/*
899	* Associates a new BlockDriverState with @blk.
900	*/
901	int blk_insert_bs(BlockBackend blk, BlockDriverState bs, Error **errp)
902	{
903	ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
904	uint64_t perm, shared_perm;
905
906	GLOBAL_STATE_CODE();
907	bdrv_ref(bs);
908	bdrv_graph_wrlock_drained();
909
910	if ((bs->open_flags & BDRV_O_INACTIVE) && blk_can_inactivate(blk)) {
911	blk->disable_perm = true;
912	perm = 0;
913	shared_perm = BLK_PERM_ALL;
914	} else {
915	perm = blk->perm;
916	shared_perm = blk->shared_perm;
917	}
918
919	blk->root = bdrv_root_attach_child(bs, "root", &child_root,
920	BDRV_CHILD_FILTERED \| BDRV_CHILD_PRIMARY,
921	perm, shared_perm, blk, errp);
922	bdrv_graph_wrunlock();
923	if (blk->root == NULL) {
924	return -EPERM;
925	}
926
927	notifier_list_notify(&blk->insert_bs_notifiers, blk);
928	if (tgm->throttle_state) {
929	throttle_group_detach_aio_context(tgm);
930	throttle_group_attach_aio_context(tgm, bdrv_get_aio_context(bs));
931	}
932
933	return 0;
934	}
935
936	/*
937	* Change BlockDriverState associated with @blk.
938	*/
939	int blk_replace_bs(BlockBackend blk, BlockDriverState new_bs, Error **errp)
940	{
941	GLOBAL_STATE_CODE();
942	return bdrv_replace_child_bs(blk->root, new_bs, errp);
943	}
944
945	/*
946	* Sets the permission bitmasks that the user of the BlockBackend needs.
947	*/
948	static int coroutine_mixed_fn GRAPH_RDLOCK
949	blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
950	Error **errp)
951	{
952	int ret;
953	GLOBAL_STATE_CODE();
954
955	if (blk->root && !blk->disable_perm) {
956	ret = bdrv_child_try_set_perm(blk->root, perm, shared_perm, errp);
957	if (ret < 0) {
958	return ret;
959	}
960	}
961
962	blk->perm = perm;
963	blk->shared_perm = shared_perm;
964
965	return 0;
966	}
967
968	int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
969	Error **errp)
970	{
971	GLOBAL_STATE_CODE();
972	GRAPH_RDLOCK_GUARD_MAINLOOP();
973
974	return blk_set_perm_locked(blk, perm, shared_perm, errp);
975	}
976
977	void blk_get_perm(BlockBackend blk, uint64_t perm, uint64_t *shared_perm)
978	{
979	GLOBAL_STATE_CODE();
980	*perm = blk->perm;
981	*shared_perm = blk->shared_perm;
982	}
983
984	/*
985	* Attach device model @dev to @blk.
986	* Return 0 on success, -EBUSY when a device model is attached already.
987	*/
988	int blk_attach_dev(BlockBackend blk, DeviceState dev)
989	{
990	GLOBAL_STATE_CODE();
991	if (blk->dev) {
992	return -EBUSY;
993	}
994
995	/* While migration is still incoming, we don't need to apply the
996	* permissions of guest device BlockBackends. We might still have a block
997	* job or NBD server writing to the image for storage migration. */
998	if (runstate_check(RUN_STATE_INMIGRATE)) {
999	blk->disable_perm = true;
1000	}
1001
1002	blk_ref(blk);
1003	blk->dev = dev;
1004	blk_iostatus_reset(blk);
1005
1006	return 0;
1007	}
1008
1009	/*
1010	* Detach device model @dev from @blk.
1011	* @dev must be currently attached to @blk.
1012	*/
1013	void blk_detach_dev(BlockBackend blk, DeviceState dev)
1014	{
1015	assert(blk->dev == dev);
1016	GLOBAL_STATE_CODE();
1017	blk->dev = NULL;
1018	blk->dev_ops = NULL;
1019	blk->dev_opaque = NULL;
1020	blk_set_perm(blk, 0, BLK_PERM_ALL, &error_abort);
1021	blk_unref(blk);
1022	}
1023
1024	/*
1025	* Return the device model attached to @blk if any, else null.
1026	*/
1027	DeviceState blk_get_attached_dev(BlockBackend blk)
1028	{
1029	GLOBAL_STATE_CODE();
1030	return blk->dev;
1031	}
1032
1033	/*
1034	* The caller is responsible for releasing the value returned
1035	* with g_free() after use.
1036	*/
1037	static char blk_get_attached_dev_id_or_path(BlockBackend blk, bool want_id)
1038	{
1039	DeviceState *dev = blk->dev;
1040	IO_CODE();
1041
1042	if (!dev) {
1043	return g_strdup("");
1044	} else if (want_id && dev->id) {
1045	return g_strdup(dev->id);
1046	}
1047
1048	return object_get_canonical_path(OBJECT(dev)) ?: g_strdup("");
1049	}
1050
1051	char blk_get_attached_dev_id(BlockBackend blk)
1052	{
1053	return blk_get_attached_dev_id_or_path(blk, true);
1054	}
1055
1056	/*
1057	* The caller is responsible for releasing the value returned
1058	* with g_free() after use.
1059	*/
1060	static char blk_get_attached_dev_path(BlockBackend blk)
1061	{
1062	return blk_get_attached_dev_id_or_path(blk, false);
1063	}
1064
1065	/*
1066	* Return the BlockBackend which has the device model @dev attached if it
1067	* exists, else null.
1068	*
1069	* @dev must not be null.
1070	*/
1071	BlockBackend blk_by_dev(void dev)
1072	{
1073	BlockBackend *blk = NULL;
1074
1075	GLOBAL_STATE_CODE();
1076
1077	assert(dev != NULL);
1078	while ((blk = blk_all_next(blk)) != NULL) {
1079	if (blk->dev == dev) {
1080	return blk;
1081	}
1082	}
1083	return NULL;
1084	}
1085
1086	/*
1087	* Set @blk's device model callbacks to @ops.
1088	* @opaque is the opaque argument to pass to the callbacks.
1089	* This is for use by device models.
1090	*/
1091	void blk_set_dev_ops(BlockBackend blk, const BlockDevOps ops,
1092	void *opaque)
1093	{
1094	GLOBAL_STATE_CODE();
1095	blk->dev_ops = ops;
1096	blk->dev_opaque = opaque;
1097
1098	/* Are we currently quiesced? Should we enforce this right now? */
1099	if (qatomic_read(&blk->quiesce_counter) && ops && ops->drained_begin) {
1100	ops->drained_begin(opaque);
1101	}
1102	}
1103
1104	/*
1105	* Notify @blk's attached device model of media change.
1106	*
1107	* If @load is true, notify of media load. This action can fail, meaning that
1108	* the medium cannot be loaded. @errp is set then.
1109	*
1110	* If @load is false, notify of media eject. This can never fail.
1111	*
1112	* Also send DEVICE_TRAY_MOVED events as appropriate.
1113	*/
1114	void blk_dev_change_media_cb(BlockBackend blk, bool load, Error *errp)
1115	{
1116	GLOBAL_STATE_CODE();
1117	if (blk->dev_ops && blk->dev_ops->change_media_cb) {
1118	bool tray_was_open, tray_is_open;
1119	Error *local_err = NULL;
1120
1121	tray_was_open = blk_dev_is_tray_open(blk);
1122	blk->dev_ops->change_media_cb(blk->dev_opaque, load, &local_err);
1123	if (local_err) {
1124	assert(load == true);
1125	error_propagate(errp, local_err);
1126	return;
1127	}
1128	tray_is_open = blk_dev_is_tray_open(blk);
1129
1130	if (tray_was_open != tray_is_open) {
1131	char *id = blk_get_attached_dev_id(blk);
1132	qapi_event_send_device_tray_moved(blk_name(blk), id, tray_is_open);
1133	g_free(id);
1134	}
1135	}
1136	}
1137
1138	static void blk_root_change_media(BdrvChild *child, bool load)
1139	{
1140	blk_dev_change_media_cb(child->opaque, load, NULL);
1141	}
1142
1143	/*
1144	* Does @blk's attached device model have removable media?
1145	* %true if no device model is attached.
1146	*/
1147	bool blk_dev_has_removable_media(BlockBackend *blk)
1148	{
1149	GLOBAL_STATE_CODE();
1150	return !blk->dev \|\| (blk->dev_ops && blk->dev_ops->change_media_cb);
1151	}
1152
1153	/*
1154	* Does @blk's attached device model have a tray?
1155	*/
1156	bool blk_dev_has_tray(BlockBackend *blk)
1157	{
1158	IO_CODE();
1159	return blk->dev_ops && blk->dev_ops->is_tray_open;
1160	}
1161
1162	/*
1163	* Notify @blk's attached device model of a media eject request.
1164	* If @force is true, the medium is about to be yanked out forcefully.
1165	*/
1166	void blk_dev_eject_request(BlockBackend *blk, bool force)
1167	{
1168	GLOBAL_STATE_CODE();
1169	if (blk->dev_ops && blk->dev_ops->eject_request_cb) {
1170	blk->dev_ops->eject_request_cb(blk->dev_opaque, force);
1171	}
1172	}
1173
1174	/*
1175	* Does @blk's attached device model have a tray, and is it open?
1176	*/
1177	bool blk_dev_is_tray_open(BlockBackend *blk)
1178	{
1179	IO_CODE();
1180	if (blk_dev_has_tray(blk)) {
1181	return blk->dev_ops->is_tray_open(blk->dev_opaque);
1182	}
1183	return false;
1184	}
1185
1186	/*
1187	* Does @blk's attached device model have the medium locked?
1188	* %false if the device model has no such lock.
1189	*/
1190	bool blk_dev_is_medium_locked(BlockBackend *blk)
1191	{
1192	GLOBAL_STATE_CODE();
1193	if (blk->dev_ops && blk->dev_ops->is_medium_locked) {
1194	return blk->dev_ops->is_medium_locked(blk->dev_opaque);
1195	}
1196	return false;
1197	}
1198
1199	/*
1200	* Notify @blk's attached device model of a backend size change.
1201	*/
1202	static void blk_root_resize(BdrvChild *child)
1203	{
1204	BlockBackend *blk = child->opaque;
1205
1206	if (blk->dev_ops && blk->dev_ops->resize_cb) {
1207	blk->dev_ops->resize_cb(blk->dev_opaque);
1208	}
1209	}
1210
1211	void blk_iostatus_enable(BlockBackend *blk)
1212	{
1213	GLOBAL_STATE_CODE();
1214	blk->iostatus_enabled = true;
1215	blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1216	}
1217
1218	/* The I/O status is only enabled if the drive explicitly
1219	* enables it _and_ the VM is configured to stop on errors */
1220	bool blk_iostatus_is_enabled(const BlockBackend *blk)
1221	{
1222	IO_CODE();
1223	return (blk->iostatus_enabled &&
1224	(blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC \|\|
1225	blk->on_write_error == BLOCKDEV_ON_ERROR_STOP \|\|
1226	blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
1227	}
1228
1229	BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
1230	{
1231	GLOBAL_STATE_CODE();
1232	return blk->iostatus;
1233	}
1234
1235	void blk_iostatus_reset(BlockBackend *blk)
1236	{
1237	GLOBAL_STATE_CODE();
1238	if (blk_iostatus_is_enabled(blk)) {
1239	blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1240	}
1241	}
1242
1243	void blk_iostatus_set_err(BlockBackend *blk, int error)
1244	{
1245	IO_CODE();
1246	assert(blk_iostatus_is_enabled(blk));
1247	if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1248	blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
1249	BLOCK_DEVICE_IO_STATUS_FAILED;
1250	}
1251	}
1252
1253	void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow)
1254	{
1255	IO_CODE();
1256	blk->allow_write_beyond_eof = allow;
1257	}
1258
1259	void blk_set_allow_aio_context_change(BlockBackend *blk, bool allow)
1260	{
1261	IO_CODE();
1262	blk->allow_aio_context_change = allow;
1263	}
1264
1265	void blk_set_disable_request_queuing(BlockBackend *blk, bool disable)
1266	{
1267	IO_CODE();
1268	qatomic_set(&blk->disable_request_queuing, disable);
1269	}
1270
1271	static int coroutine_fn GRAPH_RDLOCK
1272	blk_check_byte_request(BlockBackend *blk, int64_t offset, int64_t bytes)
1273	{
1274	int64_t len;
1275
1276	if (bytes < 0) {
1277	return -EIO;
1278	}
1279
1280	if (!blk_co_is_available(blk)) {
1281	return -ENOMEDIUM;
1282	}
1283
1284	if (offset < 0) {
1285	return -EIO;
1286	}
1287
1288	if (!blk->allow_write_beyond_eof) {
1289	len = bdrv_co_getlength(blk_bs(blk));
1290	if (len < 0) {
1291	return len;
1292	}
1293
1294	if (offset > len \|\| len - offset < bytes) {
1295	return -EIO;
1296	}
1297	}
1298
1299	return 0;
1300	}
1301
1302	/* Are we currently in a drained section? */
1303	bool blk_in_drain(BlockBackend *blk)
1304	{
1305	GLOBAL_STATE_CODE(); /* change to IO_OR_GS_CODE(), if necessary */
1306	return qatomic_read(&blk->quiesce_counter);
1307	}
1308
1309	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1310	static void coroutine_fn blk_wait_while_drained(BlockBackend *blk,
1311	BdrvRequestFlags flags)
1312	{
1313	assert(blk->in_flight > 0);
1314
1315	if (flags & BDRV_REQ_NO_QUEUE) {
1316	assert(qatomic_read(&blk->start_request_count));
1317	return;
1318	}
1319
1320	if (qatomic_read(&blk->quiesce_counter) &&
1321	!qatomic_read(&blk->disable_request_queuing)) {
1322	/*
1323	* Take lock before decrementing in flight counter so main loop thread
1324	* waits for us to enqueue ourselves before it can leave the drained
1325	* section.
1326	*/
1327	qemu_mutex_lock(&blk->queued_requests_lock);
1328	/* blk_root_drained_end() has the corresponding blk_inc_in_flight() */
1329	blk_dec_in_flight(blk);
1330	qemu_co_queue_wait(&blk->queued_requests, &blk->queued_requests_lock);
1331	qemu_mutex_unlock(&blk->queued_requests_lock);
1332	}
1333	}
1334
1335	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1336	static int coroutine_fn
1337	blk_co_do_preadv_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1338	QEMUIOVector *qiov, size_t qiov_offset,
1339	BdrvRequestFlags flags)
1340	{
1341	int ret;
1342	BlockDriverState *bs;
1343	IO_CODE();
1344
1345	blk_wait_while_drained(blk, flags);
1346	GRAPH_RDLOCK_GUARD();
1347
1348	/* Call blk_bs() only after waiting, the graph may have changed */
1349	bs = blk_bs(blk);
1350	trace_blk_co_preadv(blk, bs, offset, bytes, flags);
1351
1352	ret = blk_check_byte_request(blk, offset, bytes);
1353	if (ret < 0) {
1354	return ret;
1355	}
1356
1357	bdrv_inc_in_flight(bs);
1358
1359	/* throttling disk I/O */
1360	if (blk->public.throttle_group_member.throttle_state) {
1361	throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1362	bytes, THROTTLE_READ);
1363	}
1364
1365	ret = bdrv_co_preadv_part(blk->root, offset, bytes, qiov, qiov_offset,
1366	flags);
1367	bdrv_dec_in_flight(bs);
1368	return ret;
1369	}
1370
1371	int coroutine_fn blk_co_pread(BlockBackend *blk, int64_t offset, int64_t bytes,
1372	void *buf, BdrvRequestFlags flags)
1373	{
1374	QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1375	IO_OR_GS_CODE();
1376
1377	assert(bytes <= SIZE_MAX);
1378
1379	return blk_co_preadv(blk, offset, bytes, &qiov, flags);
1380	}
1381
1382	int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
1383	int64_t bytes, QEMUIOVector *qiov,
1384	BdrvRequestFlags flags)
1385	{
1386	int ret;
1387	IO_OR_GS_CODE();
1388
1389	blk_inc_in_flight(blk);
1390	ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, 0, flags);
1391	blk_dec_in_flight(blk);
1392
1393	return ret;
1394	}
1395
1396	int coroutine_fn blk_co_preadv_part(BlockBackend *blk, int64_t offset,
1397	int64_t bytes, QEMUIOVector *qiov,
1398	size_t qiov_offset, BdrvRequestFlags flags)
1399	{
1400	int ret;
1401	IO_OR_GS_CODE();
1402
1403	blk_inc_in_flight(blk);
1404	ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, qiov_offset, flags);
1405	blk_dec_in_flight(blk);
1406
1407	return ret;
1408	}
1409
1410	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1411	static int coroutine_fn
1412	blk_co_do_pwritev_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1413	QEMUIOVector *qiov, size_t qiov_offset,
1414	BdrvRequestFlags flags)
1415	{
1416	int ret;
1417	BlockDriverState *bs;
1418	IO_CODE();
1419
1420	blk_wait_while_drained(blk, flags);
1421	GRAPH_RDLOCK_GUARD();
1422
1423	/* Call blk_bs() only after waiting, the graph may have changed */
1424	bs = blk_bs(blk);
1425	trace_blk_co_pwritev(blk, bs, offset, bytes, flags);
1426
1427	ret = blk_check_byte_request(blk, offset, bytes);
1428	if (ret < 0) {
1429	return ret;
1430	}
1431
1432	bdrv_inc_in_flight(bs);
1433	/* throttling disk I/O */
1434	if (blk->public.throttle_group_member.throttle_state) {
1435	throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1436	bytes, THROTTLE_WRITE);
1437	}
1438
1439	if (!blk->enable_write_cache) {
1440	flags \|= BDRV_REQ_FUA;
1441	}
1442
1443	ret = bdrv_co_pwritev_part(blk->root, offset, bytes, qiov, qiov_offset,
1444	flags);
1445	bdrv_dec_in_flight(bs);
1446	return ret;
1447	}
1448
1449	int coroutine_fn blk_co_pwritev_part(BlockBackend *blk, int64_t offset,
1450	int64_t bytes,
1451	QEMUIOVector *qiov, size_t qiov_offset,
1452	BdrvRequestFlags flags)
1453	{
1454	int ret;
1455	IO_OR_GS_CODE();
1456
1457	blk_inc_in_flight(blk);
1458	ret = blk_co_do_pwritev_part(blk, offset, bytes, qiov, qiov_offset, flags);
1459	blk_dec_in_flight(blk);
1460
1461	return ret;
1462	}
1463
1464	int coroutine_fn blk_co_pwrite(BlockBackend *blk, int64_t offset, int64_t bytes,
1465	const void *buf, BdrvRequestFlags flags)
1466	{
1467	QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1468	IO_OR_GS_CODE();
1469
1470	assert(bytes <= SIZE_MAX);
1471
1472	return blk_co_pwritev(blk, offset, bytes, &qiov, flags);
1473	}
1474
1475	int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
1476	int64_t bytes, QEMUIOVector *qiov,
1477	BdrvRequestFlags flags)
1478	{
1479	IO_OR_GS_CODE();
1480	return blk_co_pwritev_part(blk, offset, bytes, qiov, 0, flags);
1481	}
1482
1483	int coroutine_fn blk_co_block_status_above(BlockBackend *blk,
1484	BlockDriverState *base,
1485	int64_t offset, int64_t bytes,
1486	int64_t pnum, int64_t map,
1487	BlockDriverState **file)
1488	{
1489	IO_CODE();
1490	GRAPH_RDLOCK_GUARD();
1491	return bdrv_co_block_status_above(blk_bs(blk), base, offset, bytes, pnum,
1492	map, file);
1493	}
1494
1495	int coroutine_fn blk_co_is_allocated_above(BlockBackend *blk,
1496	BlockDriverState *base,
1497	bool include_base, int64_t offset,
1498	int64_t bytes, int64_t *pnum)
1499	{
1500	IO_CODE();
1501	GRAPH_RDLOCK_GUARD();
1502	return bdrv_co_is_allocated_above(blk_bs(blk), base, include_base, offset,
1503	bytes, pnum);
1504	}
1505
1506	typedef struct BlkRwCo {
1507	BlockBackend *blk;
1508	int64_t offset;
1509	void *iobuf;
1510	int ret;
1511	BdrvRequestFlags flags;
1512	} BlkRwCo;
1513
1514	int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags)
1515	{
1516	GLOBAL_STATE_CODE();
1517	return bdrv_make_zero(blk->root, flags);
1518	}
1519
1520	void blk_inc_in_flight(BlockBackend *blk)
1521	{
1522	IO_CODE();
1523	qatomic_inc(&blk->in_flight);
1524	}
1525
1526	void blk_dec_in_flight(BlockBackend *blk)
1527	{
1528	IO_CODE();
1529	qatomic_dec(&blk->in_flight);
1530	aio_wait_kick();
1531	}
1532
1533	void coroutine_fn blk_co_start_request(BlockBackend *blk)
1534	{
1535	blk_inc_in_flight(blk);
1536	blk_wait_while_drained(blk, 0);
1537	qatomic_inc(&blk->start_request_count);
1538	}
1539
1540	void blk_end_request(BlockBackend *blk)
1541	{
1542	qatomic_dec(&blk->start_request_count);
1543	blk_dec_in_flight(blk);
1544	}
1545
1546	static void error_callback_bh(void *opaque)
1547	{
1548	struct BlockBackendAIOCB *acb = opaque;
1549
1550	blk_dec_in_flight(acb->blk);
1551	acb->common.cb(acb->common.opaque, acb->ret);
1552	qemu_aio_unref(acb);
1553	}
1554
1555	BlockAIOCB blk_abort_aio_request(BlockBackend blk,
1556	BlockCompletionFunc *cb,
1557	void *opaque, int ret)
1558	{
1559	struct BlockBackendAIOCB *acb;
1560	IO_CODE();
1561
1562	blk_inc_in_flight(blk);
1563	acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
1564	acb->blk = blk;
1565	acb->ret = ret;
1566
1567	replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1568	error_callback_bh, acb);
1569	return &acb->common;
1570	}
1571
1572	typedef struct BlkAioEmAIOCB {
1573	BlockAIOCB common;
1574	BlkRwCo rwco;
1575	int64_t bytes;
1576	bool has_returned;
1577	} BlkAioEmAIOCB;
1578
1579	static const AIOCBInfo blk_aio_em_aiocb_info = {
1580	.aiocb_size = sizeof(BlkAioEmAIOCB),
1581	};
1582
1583	static void blk_aio_complete(BlkAioEmAIOCB *acb)
1584	{
1585	if (acb->has_returned) {
1586	acb->common.cb(acb->common.opaque, acb->rwco.ret);
1587	blk_dec_in_flight(acb->rwco.blk);
1588	qemu_aio_unref(acb);
1589	}
1590	}
1591
1592	static void blk_aio_complete_bh(void *opaque)
1593	{
1594	BlkAioEmAIOCB *acb = opaque;
1595	assert(acb->has_returned);
1596	blk_aio_complete(acb);
1597	}
1598
1599	static BlockAIOCB blk_aio_prwv(BlockBackend blk, int64_t offset,
1600	int64_t bytes,
1601	void *iobuf, CoroutineEntry co_entry,
1602	BdrvRequestFlags flags,
1603	BlockCompletionFunc cb, void opaque)
1604	{
1605	BlkAioEmAIOCB *acb;
1606	Coroutine *co;
1607
1608	blk_inc_in_flight(blk);
1609	acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1610	acb->rwco = (BlkRwCo) {
1611	.blk = blk,
1612	.offset = offset,
1613	.iobuf = iobuf,
1614	.flags = flags,
1615	.ret = NOT_DONE,
1616	};
1617	acb->bytes = bytes;
1618	acb->has_returned = false;
1619
1620	co = qemu_coroutine_create(co_entry, acb);
1621	aio_co_enter(qemu_get_current_aio_context(), co);
1622
1623	acb->has_returned = true;
1624	if (acb->rwco.ret != NOT_DONE) {
1625	replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1626	blk_aio_complete_bh, acb);
1627	}
1628
1629	return &acb->common;
1630	}
1631
1632	static void coroutine_fn blk_aio_read_entry(void *opaque)
1633	{
1634	BlkAioEmAIOCB *acb = opaque;
1635	BlkRwCo *rwco = &acb->rwco;
1636	QEMUIOVector *qiov = rwco->iobuf;
1637
1638	assert(qiov->size == acb->bytes);
1639	rwco->ret = blk_co_do_preadv_part(rwco->blk, rwco->offset, acb->bytes, qiov,
1640	0, rwco->flags);
1641	blk_aio_complete(acb);
1642	}
1643
1644	static void coroutine_fn blk_aio_write_entry(void *opaque)
1645	{
1646	BlkAioEmAIOCB *acb = opaque;
1647	BlkRwCo *rwco = &acb->rwco;
1648	QEMUIOVector *qiov = rwco->iobuf;
1649
1650	assert(!qiov \|\| qiov->size == acb->bytes);
1651	rwco->ret = blk_co_do_pwritev_part(rwco->blk, rwco->offset, acb->bytes,
1652	qiov, 0, rwco->flags);
1653	blk_aio_complete(acb);
1654	}
1655
1656	BlockAIOCB blk_aio_pwrite_zeroes(BlockBackend blk, int64_t offset,
1657	int64_t bytes, BdrvRequestFlags flags,
1658	BlockCompletionFunc cb, void opaque)
1659	{
1660	IO_CODE();
1661	return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_write_entry,
1662	flags \| BDRV_REQ_ZERO_WRITE, cb, opaque);
1663	}
1664
1665	int64_t coroutine_fn blk_co_getlength(BlockBackend *blk)
1666	{
1667	IO_CODE();
1668	GRAPH_RDLOCK_GUARD();
1669
1670	if (!blk_co_is_available(blk)) {
1671	return -ENOMEDIUM;
1672	}
1673
1674	return bdrv_co_getlength(blk_bs(blk));
1675	}
1676
1677	int64_t coroutine_fn blk_co_nb_sectors(BlockBackend *blk)
1678	{
1679	BlockDriverState *bs = blk_bs(blk);
1680
1681	IO_CODE();
1682	GRAPH_RDLOCK_GUARD();
1683
1684	if (!bs) {
1685	return -ENOMEDIUM;
1686	} else {
1687	return bdrv_co_nb_sectors(bs);
1688	}
1689	}
1690
1691	/*
1692	* This wrapper is written by hand because this function is in the hot I/O path,
1693	* via blk_get_geometry.
1694	*/
1695	int64_t coroutine_mixed_fn blk_nb_sectors(BlockBackend *blk)
1696	{
1697	BlockDriverState *bs = blk_bs(blk);
1698
1699	IO_CODE();
1700
1701	if (!bs) {
1702	return -ENOMEDIUM;
1703	} else {
1704	return bdrv_nb_sectors(bs);
1705	}
1706	}
1707
1708	/* return 0 as number of sectors if no device present or error */
1709	void coroutine_fn blk_co_get_geometry(BlockBackend *blk,
1710	uint64_t *nb_sectors_ptr)
1711	{
1712	int64_t ret = blk_co_nb_sectors(blk);
1713	*nb_sectors_ptr = ret < 0 ? 0 : ret;
1714	}
1715
1716	/*
1717	* This wrapper is written by hand because this function is in the hot I/O path.
1718	*/
1719	void coroutine_mixed_fn blk_get_geometry(BlockBackend *blk,
1720	uint64_t *nb_sectors_ptr)
1721	{
1722	int64_t ret = blk_nb_sectors(blk);
1723	*nb_sectors_ptr = ret < 0 ? 0 : ret;
1724	}
1725
1726	BlockAIOCB blk_aio_preadv(BlockBackend blk, int64_t offset,
1727	QEMUIOVector *qiov, BdrvRequestFlags flags,
1728	BlockCompletionFunc cb, void opaque)
1729	{
1730	IO_CODE();
1731	assert((uint64_t)qiov->size <= INT64_MAX);
1732	return blk_aio_prwv(blk, offset, qiov->size, qiov,
1733	blk_aio_read_entry, flags, cb, opaque);
1734	}
1735
1736	BlockAIOCB blk_aio_pwritev(BlockBackend blk, int64_t offset,
1737	QEMUIOVector *qiov, BdrvRequestFlags flags,
1738	BlockCompletionFunc cb, void opaque)
1739	{
1740	IO_CODE();
1741	assert((uint64_t)qiov->size <= INT64_MAX);
1742	return blk_aio_prwv(blk, offset, qiov->size, qiov,
1743	blk_aio_write_entry, flags, cb, opaque);
1744	}
1745
1746	void blk_aio_cancel(BlockAIOCB *acb)
1747	{
1748	GLOBAL_STATE_CODE();
1749	bdrv_aio_cancel(acb);
1750	}
1751
1752	void blk_aio_cancel_async(BlockAIOCB *acb)
1753	{
1754	IO_CODE();
1755	bdrv_aio_cancel_async(acb);
1756	}
1757
1758	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1759	static int coroutine_fn
1760	blk_co_do_ioctl(BlockBackend blk, unsigned long int req, void buf)
1761	{
1762	IO_CODE();
1763
1764	blk_wait_while_drained(blk, 0);
1765	GRAPH_RDLOCK_GUARD();
1766
1767	if (!blk_co_is_available(blk)) {
1768	return -ENOMEDIUM;
1769	}
1770
1771	return bdrv_co_ioctl(blk_bs(blk), req, buf);
1772	}
1773
1774	int coroutine_fn blk_co_ioctl(BlockBackend *blk, unsigned long int req,
1775	void *buf)
1776	{
1777	int ret;
1778	IO_OR_GS_CODE();
1779
1780	blk_inc_in_flight(blk);
1781	ret = blk_co_do_ioctl(blk, req, buf);
1782	blk_dec_in_flight(blk);
1783
1784	return ret;
1785	}
1786
1787	static void coroutine_fn blk_aio_ioctl_entry(void *opaque)
1788	{
1789	BlkAioEmAIOCB *acb = opaque;
1790	BlkRwCo *rwco = &acb->rwco;
1791
1792	rwco->ret = blk_co_do_ioctl(rwco->blk, rwco->offset, rwco->iobuf);
1793
1794	blk_aio_complete(acb);
1795	}
1796
1797	BlockAIOCB blk_aio_ioctl(BlockBackend blk, unsigned long int req, void *buf,
1798	BlockCompletionFunc cb, void opaque)
1799	{
1800	IO_CODE();
1801	return blk_aio_prwv(blk, req, 0, buf, blk_aio_ioctl_entry, 0, cb, opaque);
1802	}
1803
1804	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1805	static int coroutine_fn
1806	blk_co_do_pdiscard(BlockBackend *blk, int64_t offset, int64_t bytes,
1807	BdrvRequestFlags flags)
1808	{
1809	int ret;
1810	IO_CODE();
1811
1812	blk_wait_while_drained(blk, flags);
1813	GRAPH_RDLOCK_GUARD();
1814
1815	ret = blk_check_byte_request(blk, offset, bytes);
1816	if (ret < 0) {
1817	return ret;
1818	}
1819
1820	return bdrv_co_pdiscard(blk->root, offset, bytes);
1821	}
1822
1823	static void coroutine_fn blk_aio_pdiscard_entry(void *opaque)
1824	{
1825	BlkAioEmAIOCB *acb = opaque;
1826	BlkRwCo *rwco = &acb->rwco;
1827
1828	rwco->ret = blk_co_do_pdiscard(rwco->blk, rwco->offset, acb->bytes, 0);
1829	blk_aio_complete(acb);
1830	}
1831
1832	BlockAIOCB blk_aio_pdiscard(BlockBackend blk,
1833	int64_t offset, int64_t bytes,
1834	BlockCompletionFunc cb, void opaque)
1835	{
1836	IO_CODE();
1837	return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_pdiscard_entry, 0,
1838	cb, opaque);
1839	}
1840
1841	int coroutine_fn blk_co_pdiscard(BlockBackend *blk, int64_t offset,
1842	int64_t bytes, BdrvRequestFlags flags)
1843	{
1844	int ret;
1845	IO_OR_GS_CODE();
1846
1847	blk_inc_in_flight(blk);
1848	ret = blk_co_do_pdiscard(blk, offset, bytes, flags);
1849	blk_dec_in_flight(blk);
1850
1851	return ret;
1852	}
1853
1854	/* To be called between exactly one pair of blk_inc/dec_in_flight() */
1855	static int coroutine_fn blk_co_do_flush(BlockBackend *blk)
1856	{
1857	IO_CODE();
1858	blk_wait_while_drained(blk, 0);
1859	GRAPH_RDLOCK_GUARD();
1860
1861	if (!blk_co_is_available(blk)) {
1862	return -ENOMEDIUM;
1863	}
1864
1865	return bdrv_co_flush(blk_bs(blk));
1866	}
1867
1868	static void coroutine_fn blk_aio_flush_entry(void *opaque)
1869	{
1870	BlkAioEmAIOCB *acb = opaque;
1871	BlkRwCo *rwco = &acb->rwco;
1872
1873	rwco->ret = blk_co_do_flush(rwco->blk);
1874	blk_aio_complete(acb);
1875	}
1876
1877	BlockAIOCB blk_aio_flush(BlockBackend blk,
1878	BlockCompletionFunc cb, void opaque)
1879	{
1880	IO_CODE();
1881	return blk_aio_prwv(blk, 0, 0, NULL, blk_aio_flush_entry, 0, cb, opaque);
1882	}
1883
1884	int coroutine_fn blk_co_flush(BlockBackend *blk)
1885	{
1886	int ret;
1887	IO_OR_GS_CODE();
1888
1889	blk_inc_in_flight(blk);
1890	ret = blk_co_do_flush(blk);
1891	blk_dec_in_flight(blk);
1892
1893	return ret;
1894	}
1895
1896	static void coroutine_fn blk_aio_zone_report_entry(void *opaque)
1897	{
1898	BlkAioEmAIOCB *acb = opaque;
1899	BlkRwCo *rwco = &acb->rwco;
1900
1901	rwco->ret = blk_co_zone_report(rwco->blk, rwco->offset,
1902	(unsigned int*)(uintptr_t)acb->bytes,
1903	rwco->iobuf);
1904	blk_aio_complete(acb);
1905	}
1906
1907	BlockAIOCB blk_aio_zone_report(BlockBackend blk, int64_t offset,
1908	unsigned int *nr_zones,
1909	BlockZoneDescriptor *zones,
1910	BlockCompletionFunc cb, void opaque)
1911	{
1912	BlkAioEmAIOCB *acb;
1913	Coroutine *co;
1914	IO_CODE();
1915
1916	blk_inc_in_flight(blk);
1917	acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1918	acb->rwco = (BlkRwCo) {
1919	.blk = blk,
1920	.offset = offset,
1921	.iobuf = zones,
1922	.ret = NOT_DONE,
1923	};
1924	acb->bytes = (int64_t)(uintptr_t)nr_zones,
1925	acb->has_returned = false;
1926
1927	co = qemu_coroutine_create(blk_aio_zone_report_entry, acb);
1928	aio_co_enter(qemu_get_current_aio_context(), co);
1929
1930	acb->has_returned = true;
1931	if (acb->rwco.ret != NOT_DONE) {
1932	replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1933	blk_aio_complete_bh, acb);
1934	}
1935
1936	return &acb->common;
1937	}
1938
1939	static void coroutine_fn blk_aio_zone_mgmt_entry(void *opaque)
1940	{
1941	BlkAioEmAIOCB *acb = opaque;
1942	BlkRwCo *rwco = &acb->rwco;
1943
1944	rwco->ret = blk_co_zone_mgmt(rwco->blk,
1945	(BlockZoneOp)(uintptr_t)rwco->iobuf,
1946	rwco->offset, acb->bytes);
1947	blk_aio_complete(acb);
1948	}
1949
1950	BlockAIOCB blk_aio_zone_mgmt(BlockBackend blk, BlockZoneOp op,
1951	int64_t offset, int64_t len,
1952	BlockCompletionFunc cb, void opaque) {
1953	BlkAioEmAIOCB *acb;
1954	Coroutine *co;
1955	IO_CODE();
1956
1957	blk_inc_in_flight(blk);
1958	acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1959	acb->rwco = (BlkRwCo) {
1960	.blk = blk,
1961	.offset = offset,
1962	.iobuf = (void *)(uintptr_t)op,
1963	.ret = NOT_DONE,
1964	};
1965	acb->bytes = len;
1966	acb->has_returned = false;
1967
1968	co = qemu_coroutine_create(blk_aio_zone_mgmt_entry, acb);
1969	aio_co_enter(qemu_get_current_aio_context(), co);
1970
1971	acb->has_returned = true;
1972	if (acb->rwco.ret != NOT_DONE) {
1973	replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1974	blk_aio_complete_bh, acb);
1975	}
1976
1977	return &acb->common;
1978	}
1979
1980	static void coroutine_fn blk_aio_zone_append_entry(void *opaque)
1981	{
1982	BlkAioEmAIOCB *acb = opaque;
1983	BlkRwCo *rwco = &acb->rwco;
1984
1985	rwco->ret = blk_co_zone_append(rwco->blk, (int64_t *)(uintptr_t)acb->bytes,
1986	rwco->iobuf, rwco->flags);
1987	blk_aio_complete(acb);
1988	}
1989
1990	BlockAIOCB blk_aio_zone_append(BlockBackend blk, int64_t *offset,
1991	QEMUIOVector *qiov, BdrvRequestFlags flags,
1992	BlockCompletionFunc cb, void opaque) {
1993	BlkAioEmAIOCB *acb;
1994	Coroutine *co;
1995	IO_CODE();
1996
1997	blk_inc_in_flight(blk);
1998	acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1999	acb->rwco = (BlkRwCo) {
2000	.blk = blk,
2001	.ret = NOT_DONE,
2002	.flags = flags,
2003	.iobuf = qiov,
2004	};
2005	acb->bytes = (int64_t)(uintptr_t)offset;
2006	acb->has_returned = false;
2007
2008	co = qemu_coroutine_create(blk_aio_zone_append_entry, acb);
2009	aio_co_enter(qemu_get_current_aio_context(), co);
2010	acb->has_returned = true;
2011	if (acb->rwco.ret != NOT_DONE) {
2012	replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
2013	blk_aio_complete_bh, acb);
2014	}
2015
2016	return &acb->common;
2017	}
2018
2019	/*
2020	* Send a zone_report command.
2021	* offset is a byte offset from the start of the device. No alignment
2022	* required for offset.
2023	* nr_zones represents IN maximum and OUT actual.
2024	*/
2025	int coroutine_fn blk_co_zone_report(BlockBackend *blk, int64_t offset,
2026	unsigned int *nr_zones,
2027	BlockZoneDescriptor *zones)
2028	{
2029	int ret;
2030	IO_CODE();
2031
2032	blk_inc_in_flight(blk); /* increase before waiting */
2033	blk_wait_while_drained(blk, 0);
2034	GRAPH_RDLOCK_GUARD();
2035	if (!blk_is_available(blk)) {
2036	blk_dec_in_flight(blk);
2037	return -ENOMEDIUM;
2038	}
2039	ret = bdrv_co_zone_report(blk_bs(blk), offset, nr_zones, zones);
2040	blk_dec_in_flight(blk);
2041	return ret;
2042	}
2043
2044	/*
2045	* Send a zone_management command.
2046	* op is the zone operation;
2047	* offset is the byte offset from the start of the zoned device;
2048	* len is the maximum number of bytes the command should operate on. It
2049	* should be aligned with the device zone size.
2050	*/
2051	int coroutine_fn blk_co_zone_mgmt(BlockBackend *blk, BlockZoneOp op,
2052	int64_t offset, int64_t len)
2053	{
2054	int ret;
2055	IO_CODE();
2056
2057	blk_inc_in_flight(blk);
2058	blk_wait_while_drained(blk, 0);
2059	GRAPH_RDLOCK_GUARD();
2060
2061	ret = blk_check_byte_request(blk, offset, len);
2062	if (ret < 0) {
2063	blk_dec_in_flight(blk);
2064	return ret;
2065	}
2066
2067	ret = bdrv_co_zone_mgmt(blk_bs(blk), op, offset, len);
2068	blk_dec_in_flight(blk);
2069	return ret;
2070	}
2071
2072	/*
2073	* Send a zone_append command.
2074	*/
2075	int coroutine_fn blk_co_zone_append(BlockBackend blk, int64_t offset,
2076	QEMUIOVector *qiov, BdrvRequestFlags flags)
2077	{
2078	int ret;
2079	IO_CODE();
2080
2081	blk_inc_in_flight(blk);
2082	blk_wait_while_drained(blk, flags);
2083	GRAPH_RDLOCK_GUARD();
2084	if (!blk_is_available(blk)) {
2085	blk_dec_in_flight(blk);
2086	return -ENOMEDIUM;
2087	}
2088
2089	ret = bdrv_co_zone_append(blk_bs(blk), offset, qiov, flags);
2090	blk_dec_in_flight(blk);
2091	return ret;
2092	}
2093
2094	void blk_drain(BlockBackend *blk)
2095	{
2096	BlockDriverState *bs = blk_bs(blk);
2097	GLOBAL_STATE_CODE();
2098
2099	if (bs) {
2100	bdrv_ref(bs);
2101	bdrv_drained_begin(bs);
2102	}
2103
2104	/* We may have -ENOMEDIUM completions in flight */
2105	AIO_WAIT_WHILE(blk_get_aio_context(blk),
2106	qatomic_read(&blk->in_flight) > 0);
2107
2108	if (bs) {
2109	bdrv_drained_end(bs);
2110	bdrv_unref(bs);
2111	}
2112	}
2113
2114	void blk_drain_all(void)
2115	{
2116	BlockBackend *blk = NULL;
2117
2118	GLOBAL_STATE_CODE();
2119
2120	bdrv_drain_all_begin();
2121
2122	while ((blk = blk_all_next(blk)) != NULL) {
2123	/* We may have -ENOMEDIUM completions in flight */
2124	AIO_WAIT_WHILE_UNLOCKED(NULL, qatomic_read(&blk->in_flight) > 0);
2125	}
2126
2127	bdrv_drain_all_end();
2128	}
2129
2130	void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
2131	BlockdevOnError on_write_error)
2132	{
2133	GLOBAL_STATE_CODE();
2134	blk->on_read_error = on_read_error;
2135	blk->on_write_error = on_write_error;
2136	}
2137
2138	BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
2139	{
2140	IO_CODE();
2141	return is_read ? blk->on_read_error : blk->on_write_error;
2142	}
2143
2144	BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
2145	int error)
2146	{
2147	BlockdevOnError on_err = blk_get_on_error(blk, is_read);
2148	IO_CODE();
2149
2150	switch (on_err) {
2151	case BLOCKDEV_ON_ERROR_ENOSPC:
2152	return (error == ENOSPC) ?
2153	BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
2154	case BLOCKDEV_ON_ERROR_STOP:
2155	return BLOCK_ERROR_ACTION_STOP;
2156	case BLOCKDEV_ON_ERROR_REPORT:
2157	return BLOCK_ERROR_ACTION_REPORT;
2158	case BLOCKDEV_ON_ERROR_IGNORE:
2159	return BLOCK_ERROR_ACTION_IGNORE;
2160	case BLOCKDEV_ON_ERROR_AUTO:
2161	default:
2162	abort();
2163	}
2164	}
2165
2166	static void send_qmp_error_event(BlockBackend *blk,
2167	BlockErrorAction action,
2168	bool is_read, int error)
2169	{
2170	IoOperationType optype;
2171	BlockDriverState *bs = blk_bs(blk);
2172	g_autofree char *path = blk_get_attached_dev_path(blk);
2173
2174	optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
2175	qapi_event_send_block_io_error(path, blk_name(blk),
2176	bs ? bdrv_get_node_name(bs) : NULL, optype,
2177	action, blk_iostatus_is_enabled(blk),
2178	error == ENOSPC, strerror(error));
2179	}
2180
2181	/* This is done by device models because, while the block layer knows
2182	* about the error, it does not know whether an operation comes from
2183	* the device or the block layer (from a job, for example).
2184	*/
2185	void blk_error_action(BlockBackend *blk, BlockErrorAction action,
2186	bool is_read, int error)
2187	{
2188	assert(error >= 0);
2189	IO_CODE();
2190
2191	if (action == BLOCK_ERROR_ACTION_STOP) {
2192	/* First set the iostatus, so that "info block" returns an iostatus
2193	* that matches the events raised so far (an additional error iostatus
2194	* is fine, but not a lost one).
2195	*/
2196	blk_iostatus_set_err(blk, error);
2197
2198	/* Then raise the request to stop the VM and the event.
2199	* qemu_system_vmstop_request_prepare has two effects. First,
2200	* it ensures that the STOP event always comes after the
2201	* BLOCK_IO_ERROR event. Second, it ensures that even if management
2202	* can observe the STOP event and do a "cont" before the STOP
2203	* event is issued, the VM will not stop. In this case, vm_start()
2204	* also ensures that the STOP/RESUME pair of events is emitted.
2205	*/
2206	qemu_system_vmstop_request_prepare();
2207	send_qmp_error_event(blk, action, is_read, error);
2208	qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
2209	} else {
2210	send_qmp_error_event(blk, action, is_read, error);
2211	}
2212	}
2213
2214	/*
2215	* Returns true if the BlockBackend can support taking write permissions
2216	* (because its root node is not read-only).
2217	*/
2218	bool blk_supports_write_perm(BlockBackend *blk)
2219	{
2220	BlockDriverState *bs = blk_bs(blk);
2221	GLOBAL_STATE_CODE();
2222
2223	if (bs) {
2224	return !bdrv_is_read_only(bs);
2225	} else {
2226	return blk->root_state.open_flags & BDRV_O_RDWR;
2227	}
2228	}
2229
2230	/*
2231	* Returns true if the BlockBackend can be written to in its current
2232	* configuration (i.e. if write permission have been requested)
2233	*/
2234	bool blk_is_writable(BlockBackend *blk)
2235	{
2236	IO_CODE();
2237	return blk->perm & BLK_PERM_WRITE;
2238	}
2239
2240	bool blk_is_sg(BlockBackend *blk)
2241	{
2242	BlockDriverState *bs = blk_bs(blk);
2243	GLOBAL_STATE_CODE();
2244
2245	if (!bs) {
2246	return false;
2247	}
2248
2249	return bdrv_is_sg(bs);
2250	}
2251
2252	bool blk_enable_write_cache(BlockBackend *blk)
2253	{
2254	IO_CODE();
2255	return blk->enable_write_cache;
2256	}
2257
2258	void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
2259	{
2260	IO_CODE();
2261	blk->enable_write_cache = wce;
2262	}
2263
2264	bool coroutine_fn blk_co_is_inserted(BlockBackend *blk)
2265	{
2266	BlockDriverState *bs = blk_bs(blk);
2267	IO_CODE();
2268	assert_bdrv_graph_readable();
2269
2270	return bs && bdrv_co_is_inserted(bs);
2271	}
2272
2273	bool coroutine_fn blk_co_is_available(BlockBackend *blk)
2274	{
2275	IO_CODE();
2276	return blk_co_is_inserted(blk) && !blk_dev_is_tray_open(blk);
2277	}
2278
2279	void coroutine_fn blk_co_lock_medium(BlockBackend *blk, bool locked)
2280	{
2281	BlockDriverState *bs = blk_bs(blk);
2282	IO_CODE();
2283	GRAPH_RDLOCK_GUARD();
2284
2285	if (bs) {
2286	bdrv_co_lock_medium(bs, locked);
2287	}
2288	}
2289
2290	void coroutine_fn blk_co_eject(BlockBackend *blk, bool eject_flag)
2291	{
2292	BlockDriverState *bs = blk_bs(blk);
2293	char *id;
2294	IO_CODE();
2295	GRAPH_RDLOCK_GUARD();
2296
2297	if (bs) {
2298	bdrv_co_eject(bs, eject_flag);
2299	}
2300
2301	/* Whether or not we ejected on the backend,
2302	* the frontend experienced a tray event. */
2303	id = blk_get_attached_dev_id(blk);
2304	qapi_event_send_device_tray_moved(blk_name(blk), id,
2305	eject_flag);
2306	g_free(id);
2307	}
2308
2309	int blk_get_flags(BlockBackend *blk)
2310	{
2311	BlockDriverState *bs = blk_bs(blk);
2312	GLOBAL_STATE_CODE();
2313
2314	if (bs) {
2315	return bdrv_get_flags(bs);
2316	} else {
2317	return blk->root_state.open_flags;
2318	}
2319	}
2320
2321	/* Returns the minimum request alignment, in bytes; guaranteed nonzero */
2322	uint32_t blk_get_request_alignment(BlockBackend *blk)
2323	{
2324	BlockDriverState *bs = blk_bs(blk);
2325	IO_CODE();
2326	return bs ? bs->bl.request_alignment : BDRV_SECTOR_SIZE;
2327	}
2328
2329	/* Returns the optimal write zeroes alignment, in bytes; guaranteed nonzero */
2330	uint32_t blk_get_pwrite_zeroes_alignment(BlockBackend *blk)
2331	{
2332	BlockDriverState *bs = blk_bs(blk);
2333	IO_CODE();
2334	if (!bs) {
2335	return BDRV_SECTOR_SIZE;
2336	}
2337	return bs->bl.pwrite_zeroes_alignment ?: bs->bl.request_alignment;
2338	}
2339
2340	/* Returns the maximum hardware transfer length, in bytes; guaranteed nonzero */
2341	uint64_t blk_get_max_hw_transfer(BlockBackend *blk)
2342	{
2343	BlockDriverState *bs = blk_bs(blk);
2344	uint64_t max = INT_MAX;
2345	IO_CODE();
2346
2347	if (bs) {
2348	max = MIN_NON_ZERO(max, bs->bl.max_hw_transfer);
2349	max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2350	}
2351	return ROUND_DOWN(max, blk_get_request_alignment(blk));
2352	}
2353
2354	/* Returns the maximum transfer length, in bytes; guaranteed nonzero */
2355	uint32_t blk_get_max_transfer(BlockBackend *blk)
2356	{
2357	BlockDriverState *bs = blk_bs(blk);
2358	uint32_t max = INT_MAX;
2359	IO_CODE();
2360
2361	if (bs) {
2362	max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2363	}
2364	return ROUND_DOWN(max, blk_get_request_alignment(blk));
2365	}
2366
2367	int blk_get_max_hw_iov(BlockBackend *blk)
2368	{
2369	IO_CODE();
2370	return MIN_NON_ZERO(blk->root->bs->bl.max_hw_iov,
2371	blk->root->bs->bl.max_iov);
2372	}
2373
2374	int blk_get_max_iov(BlockBackend *blk)
2375	{
2376	IO_CODE();
2377	return blk->root->bs->bl.max_iov;
2378	}
2379
2380	void blk_try_blockalign(BlockBackend blk, size_t size)
2381	{
2382	IO_CODE();
2383	return qemu_try_blockalign(blk ? blk_bs(blk) : NULL, size);
2384	}
2385
2386	void blk_blockalign(BlockBackend blk, size_t size)
2387	{
2388	IO_CODE();
2389	return qemu_blockalign(blk ? blk_bs(blk) : NULL, size);
2390	}
2391
2392
2393	/**
2394	* Return BB's current AioContext. Note that this context may change
2395	* concurrently at any time, with one exception: If the BB has a root node
2396	* attached, its context will only change through bdrv_try_change_aio_context(),
2397	* which creates a drained section. Therefore, incrementing such a BB's
2398	* in-flight counter will prevent its context from changing.
2399	*/
2400	AioContext blk_get_aio_context(BlockBackend blk)
2401	{
2402	IO_CODE();
2403
2404	if (!blk) {
2405	return qemu_get_aio_context();
2406	}
2407
2408	return qatomic_read(&blk->ctx);
2409	}
2410
2411	int blk_set_aio_context(BlockBackend blk, AioContext new_context,
2412	Error **errp)
2413	{
2414	bool old_allow_change;
2415	BlockDriverState *bs = blk_bs(blk);
2416	int ret;
2417
2418	GLOBAL_STATE_CODE();
2419
2420	if (!bs) {
2421	qatomic_set(&blk->ctx, new_context);
2422	return 0;
2423	}
2424
2425	bdrv_ref(bs);
2426
2427	old_allow_change = blk->allow_aio_context_change;
2428	blk->allow_aio_context_change = true;
2429
2430	ret = bdrv_try_change_aio_context(bs, new_context, NULL, errp);
2431
2432	blk->allow_aio_context_change = old_allow_change;
2433
2434	bdrv_unref(bs);
2435	return ret;
2436	}
2437
2438	typedef struct BdrvStateBlkRootContext {
2439	AioContext *new_ctx;
2440	BlockBackend *blk;
2441	} BdrvStateBlkRootContext;
2442
2443	static void blk_root_set_aio_ctx_commit(void *opaque)
2444	{
2445	BdrvStateBlkRootContext *s = opaque;
2446	BlockBackend *blk = s->blk;
2447	AioContext *new_context = s->new_ctx;
2448	ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2449
2450	qatomic_set(&blk->ctx, new_context);
2451	if (tgm->throttle_state) {
2452	throttle_group_detach_aio_context(tgm);
2453	throttle_group_attach_aio_context(tgm, new_context);
2454	}
2455	}
2456
2457	static TransactionActionDrv set_blk_root_context = {
2458	.commit = blk_root_set_aio_ctx_commit,
2459	.clean = g_free,
2460	};
2461
2462	static bool blk_root_change_aio_ctx(BdrvChild child, AioContext ctx,
2463	GHashTable visited, Transaction tran,
2464	Error **errp)
2465	{
2466	BlockBackend *blk = child->opaque;
2467	BdrvStateBlkRootContext *s;
2468
2469	if (!blk->allow_aio_context_change) {
2470	/*
2471	* Manually created BlockBackends (those with a name) that are not
2472	* attached to anything can change their AioContext without updating
2473	* their user; return an error for others.
2474	*/
2475	if (!blk->name \|\| blk->dev) {
2476	/* TODO Add BB name/QOM path */
2477	error_setg(errp, "Cannot change iothread of active block backend");
2478	return false;
2479	}
2480	}
2481
2482	s = g_new(BdrvStateBlkRootContext, 1);
2483	*s = (BdrvStateBlkRootContext) {
2484	.new_ctx = ctx,
2485	.blk = blk,
2486	};
2487
2488	tran_add(tran, &set_blk_root_context, s);
2489	return true;
2490	}
2491
2492	void blk_add_aio_context_notifier(BlockBackend *blk,
2493	void (attached_aio_context)(AioContext new_context, void *opaque),
2494	void (detach_aio_context)(void opaque), void *opaque)
2495	{
2496	BlockBackendAioNotifier *notifier;
2497	BlockDriverState *bs = blk_bs(blk);
2498	GLOBAL_STATE_CODE();
2499
2500	notifier = g_new(BlockBackendAioNotifier, 1);
2501	notifier->attached_aio_context = attached_aio_context;
2502	notifier->detach_aio_context = detach_aio_context;
2503	notifier->opaque = opaque;
2504	QLIST_INSERT_HEAD(&blk->aio_notifiers, notifier, list);
2505
2506	if (bs) {
2507	bdrv_add_aio_context_notifier(bs, attached_aio_context,
2508	detach_aio_context, opaque);
2509	}
2510	}
2511
2512	void blk_remove_aio_context_notifier(BlockBackend *blk,
2513	void (attached_aio_context)(AioContext ,
2514	void *),
2515	void (detach_aio_context)(void ),
2516	void *opaque)
2517	{
2518	BlockBackendAioNotifier *notifier;
2519	BlockDriverState *bs = blk_bs(blk);
2520
2521	GLOBAL_STATE_CODE();
2522
2523	if (bs) {
2524	bdrv_remove_aio_context_notifier(bs, attached_aio_context,
2525	detach_aio_context, opaque);
2526	}
2527
2528	QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
2529	if (notifier->attached_aio_context == attached_aio_context &&
2530	notifier->detach_aio_context == detach_aio_context &&
2531	notifier->opaque == opaque) {
2532	QLIST_REMOVE(notifier, list);
2533	g_free(notifier);
2534	return;
2535	}
2536	}
2537
2538	abort();
2539	}
2540
2541	void blk_add_remove_bs_notifier(BlockBackend blk, Notifier notify)
2542	{
2543	GLOBAL_STATE_CODE();
2544	notifier_list_add(&blk->remove_bs_notifiers, notify);
2545	}
2546
2547	BlockAcctStats blk_get_stats(BlockBackend blk)
2548	{
2549	IO_CODE();
2550	return &blk->stats;
2551	}
2552
2553	void blk_aio_get(const AIOCBInfo aiocb_info, BlockBackend *blk,
2554	BlockCompletionFunc cb, void opaque)
2555	{
2556	IO_CODE();
2557	return qemu_aio_get(aiocb_info, blk_bs(blk), cb, opaque);
2558	}
2559
2560	int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
2561	int64_t bytes, BdrvRequestFlags flags)
2562	{
2563	IO_OR_GS_CODE();
2564	return blk_co_pwritev(blk, offset, bytes, NULL,
2565	flags \| BDRV_REQ_ZERO_WRITE);
2566	}
2567
2568	int coroutine_fn blk_co_pwrite_compressed(BlockBackend *blk, int64_t offset,
2569	int64_t bytes, const void *buf)
2570	{
2571	QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
2572	IO_OR_GS_CODE();
2573	return blk_co_pwritev_part(blk, offset, bytes, &qiov, 0,
2574	BDRV_REQ_WRITE_COMPRESSED);
2575	}
2576
2577	int coroutine_fn blk_co_truncate(BlockBackend *blk, int64_t offset, bool exact,
2578	PreallocMode prealloc, BdrvRequestFlags flags,
2579	Error **errp)
2580	{
2581	IO_OR_GS_CODE();
2582	GRAPH_RDLOCK_GUARD();
2583	if (!blk_co_is_available(blk)) {
2584	error_setg(errp, "No medium inserted");
2585	return -ENOMEDIUM;
2586	}
2587
2588	return bdrv_co_truncate(blk->root, offset, exact, prealloc, flags, errp);
2589	}
2590
2591	int blk_save_vmstate(BlockBackend blk, const uint8_t buf,
2592	int64_t pos, int size)
2593	{
2594	int ret;
2595	GLOBAL_STATE_CODE();
2596
2597	if (!blk_is_available(blk)) {
2598	return -ENOMEDIUM;
2599	}
2600
2601	ret = bdrv_save_vmstate(blk_bs(blk), buf, pos, size);
2602	if (ret < 0) {
2603	return ret;
2604	}
2605
2606	if (ret == size && !blk->enable_write_cache) {
2607	ret = bdrv_flush(blk_bs(blk));
2608	}
2609
2610	return ret < 0 ? ret : size;
2611	}
2612
2613	int blk_load_vmstate(BlockBackend blk, uint8_t buf, int64_t pos, int size)
2614	{
2615	GLOBAL_STATE_CODE();
2616	if (!blk_is_available(blk)) {
2617	return -ENOMEDIUM;
2618	}
2619
2620	return bdrv_load_vmstate(blk_bs(blk), buf, pos, size);
2621	}
2622
2623	int blk_probe_blocksizes(BlockBackend blk, BlockSizes bsz)
2624	{
2625	GLOBAL_STATE_CODE();
2626	GRAPH_RDLOCK_GUARD_MAINLOOP();
2627
2628	if (!blk_is_available(blk)) {
2629	return -ENOMEDIUM;
2630	}
2631
2632	return bdrv_probe_blocksizes(blk_bs(blk), bsz);
2633	}
2634
2635	int blk_probe_geometry(BlockBackend blk, HDGeometry geo)
2636	{
2637	GLOBAL_STATE_CODE();
2638	if (!blk_is_available(blk)) {
2639	return -ENOMEDIUM;
2640	}
2641
2642	return bdrv_probe_geometry(blk_bs(blk), geo);
2643	}
2644
2645	/*
2646	* Updates the BlockBackendRootState object with data from the currently
2647	* attached BlockDriverState.
2648	*/
2649	void blk_update_root_state(BlockBackend *blk)
2650	{
2651	GLOBAL_STATE_CODE();
2652	assert(blk->root);
2653
2654	blk->root_state.open_flags = blk->root->bs->open_flags;
2655	blk->root_state.detect_zeroes = blk->root->bs->detect_zeroes;
2656	}
2657
2658	/*
2659	* Returns the detect-zeroes setting to be used for bdrv_open() of a
2660	* BlockDriverState which is supposed to inherit the root state.
2661	*/
2662	bool blk_get_detect_zeroes_from_root_state(BlockBackend *blk)
2663	{
2664	GLOBAL_STATE_CODE();
2665	return blk->root_state.detect_zeroes;
2666	}
2667
2668	/*
2669	* Returns the flags to be used for bdrv_open() of a BlockDriverState which is
2670	* supposed to inherit the root state.
2671	*/
2672	int blk_get_open_flags_from_root_state(BlockBackend *blk)
2673	{
2674	GLOBAL_STATE_CODE();
2675	return blk->root_state.open_flags;
2676	}
2677
2678	BlockBackendRootState blk_get_root_state(BlockBackend blk)
2679	{
2680	GLOBAL_STATE_CODE();
2681	return &blk->root_state;
2682	}
2683
2684	int blk_commit_all(void)
2685	{
2686	BlockBackend *blk = NULL;
2687	GLOBAL_STATE_CODE();
2688	GRAPH_RDLOCK_GUARD_MAINLOOP();
2689
2690	while ((blk = blk_all_next(blk)) != NULL) {
2691	BlockDriverState *unfiltered_bs = bdrv_skip_filters(blk_bs(blk));
2692
2693	if (blk_is_inserted(blk) && bdrv_cow_child(unfiltered_bs)) {
2694	int ret;
2695
2696	ret = bdrv_commit(unfiltered_bs);
2697	if (ret < 0) {
2698	return ret;
2699	}
2700	}
2701	}
2702	return 0;
2703	}
2704
2705
2706	/* throttling disk I/O limits */
2707	void blk_set_io_limits(BlockBackend blk, ThrottleConfig cfg)
2708	{
2709	GLOBAL_STATE_CODE();
2710	throttle_group_config(&blk->public.throttle_group_member, cfg);
2711	}
2712
2713	void blk_io_limits_disable(BlockBackend *blk)
2714	{
2715	BlockDriverState *bs = blk_bs(blk);
2716	ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2717	assert(tgm->throttle_state);
2718	GLOBAL_STATE_CODE();
2719	if (bs) {
2720	bdrv_ref(bs);
2721	bdrv_drained_begin(bs);
2722	}
2723	throttle_group_unregister_tgm(tgm);
2724	if (bs) {
2725	bdrv_drained_end(bs);
2726	bdrv_unref(bs);
2727	}
2728	}
2729
2730	/* should be called before blk_set_io_limits if a limit is set */
2731	void blk_io_limits_enable(BlockBackend blk, const char group)
2732	{
2733	assert(!blk->public.throttle_group_member.throttle_state);
2734	GLOBAL_STATE_CODE();
2735	throttle_group_register_tgm(&blk->public.throttle_group_member,
2736	group, blk_get_aio_context(blk));
2737	}
2738
2739	void blk_io_limits_update_group(BlockBackend blk, const char group)
2740	{
2741	GLOBAL_STATE_CODE();
2742	/* this BB is not part of any group */
2743	if (!blk->public.throttle_group_member.throttle_state) {
2744	return;
2745	}
2746
2747	/* this BB is a part of the same group than the one we want */
2748	if (!g_strcmp0(throttle_group_get_name(&blk->public.throttle_group_member),
2749	group)) {
2750	return;
2751	}
2752
2753	/* need to change the group this bs belong to */
2754	blk_io_limits_disable(blk);
2755	blk_io_limits_enable(blk, group);
2756	}
2757
2758	static void blk_root_drained_begin(BdrvChild *child)
2759	{
2760	BlockBackend *blk = child->opaque;
2761	ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2762
2763	if (qatomic_fetch_inc(&blk->quiesce_counter) == 0) {
2764	if (blk->dev_ops && blk->dev_ops->drained_begin) {
2765	blk->dev_ops->drained_begin(blk->dev_opaque);
2766	}
2767	}
2768
2769	/* Note that blk->root may not be accessible here yet if we are just
2770	* attaching to a BlockDriverState that is drained. Use child instead. */
2771
2772	if (qatomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
2773	throttle_group_restart_tgm(tgm);
2774	}
2775	}
2776
2777	static bool blk_root_drained_poll(BdrvChild *child)
2778	{
2779	BlockBackend *blk = child->opaque;
2780	bool busy = false;
2781	assert(qatomic_read(&blk->quiesce_counter));
2782
2783	if (blk->dev_ops && blk->dev_ops->drained_poll) {
2784	busy = blk->dev_ops->drained_poll(blk->dev_opaque);
2785	}
2786	return busy \|\| !!blk->in_flight;
2787	}
2788
2789	static void blk_root_drained_end(BdrvChild *child)
2790	{
2791	BlockBackend *blk = child->opaque;
2792	assert(qatomic_read(&blk->quiesce_counter));
2793
2794	assert(blk->public.throttle_group_member.io_limits_disabled);
2795	qatomic_dec(&blk->public.throttle_group_member.io_limits_disabled);
2796
2797	if (qatomic_fetch_dec(&blk->quiesce_counter) == 1) {
2798	if (blk->dev_ops && blk->dev_ops->drained_end) {
2799	blk->dev_ops->drained_end(blk->dev_opaque);
2800	}
2801	qemu_mutex_lock(&blk->queued_requests_lock);
2802	while (!qemu_co_queue_empty(&blk->queued_requests)) {
2803	/* Resume all queued requests */
2804	blk_inc_in_flight(blk);
2805	qemu_co_enter_next(&blk->queued_requests,
2806	&blk->queued_requests_lock);
2807	}
2808	qemu_mutex_unlock(&blk->queued_requests_lock);
2809	}
2810	}
2811
2812	bool blk_register_buf(BlockBackend blk, void host, size_t size, Error **errp)
2813	{
2814	BlockDriverState *bs = blk_bs(blk);
2815
2816	GLOBAL_STATE_CODE();
2817
2818	if (bs) {
2819	return bdrv_register_buf(bs, host, size, errp);
2820	}
2821	return true;
2822	}
2823
2824	void blk_unregister_buf(BlockBackend blk, void host, size_t size)
2825	{
2826	BlockDriverState *bs = blk_bs(blk);
2827
2828	GLOBAL_STATE_CODE();
2829
2830	if (bs) {
2831	bdrv_unregister_buf(bs, host, size);
2832	}
2833	}
2834
2835	int coroutine_fn blk_co_copy_range(BlockBackend *blk_in, int64_t off_in,
2836	BlockBackend *blk_out, int64_t off_out,
2837	int64_t bytes, BdrvRequestFlags read_flags,
2838	BdrvRequestFlags write_flags)
2839	{
2840	int r;
2841	IO_CODE();
2842	GRAPH_RDLOCK_GUARD();
2843
2844	r = blk_check_byte_request(blk_in, off_in, bytes);
2845	if (r) {
2846	return r;
2847	}
2848	r = blk_check_byte_request(blk_out, off_out, bytes);
2849	if (r) {
2850	return r;
2851	}
2852
2853	return bdrv_co_copy_range(blk_in->root, off_in,
2854	blk_out->root, off_out,
2855	bytes, read_flags, write_flags);
2856	}
2857
2858	const BdrvChild blk_root(BlockBackend blk)
2859	{
2860	GLOBAL_STATE_CODE();
2861	return blk->root;
2862	}
2863
2864	int blk_make_empty(BlockBackend blk, Error *errp)
2865	{
2866	GLOBAL_STATE_CODE();
2867	GRAPH_RDLOCK_GUARD_MAINLOOP();
2868
2869	if (!blk_is_available(blk)) {
2870	error_setg(errp, "No medium inserted");
2871	return -ENOMEDIUM;
2872	}
2873
2874	return bdrv_make_empty(blk->root, errp);
2875	}