file-posix.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 4,746 lines 137 KB

Raw

1	/*
2	* Block driver for RAW files (posix)
3	*
4	* Copyright (c) 2006 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	#include "qemu/osdep.h"
26	#include "qapi/error.h"
27	#include "qemu/cutils.h"
28	#include "qemu/error-report.h"
29	#include "block/block-io.h"
30	#include "block/block_int.h"
31	#include "qemu/module.h"
32	#include "qemu/option.h"
33	#include "qemu/units.h"
34	#include "qemu/memalign.h"
35	#include "trace.h"
36	#include "block/thread-pool.h"
37	#include "qemu/iov.h"
38	#include "block/raw-aio.h"
39	#include "qobject/qdict.h"
40	#include "qobject/qstring.h"
41
42	#include "scsi/pr-manager.h"
43	#include "scsi/constants.h"
44	#include "scsi/utils.h"
45
46	#ifndef __sun__
47	#include <sys/ioctl.h>
48	#endif
49
50	#if defined(__APPLE__) && (__MACH__) && defined(HAVE_HOST_BLOCK_DEVICE)
51	#include <paths.h>
52	#include <sys/param.h>
53	#include <sys/mount.h>
54	#include <IOKit/IOKitLib.h>
55	#include <IOKit/IOBSD.h>
56	#include <IOKit/storage/IOMediaBSDClient.h>
57	#include <IOKit/storage/IOMedia.h>
58	#include <IOKit/storage/IOCDMedia.h>
59	//#include <IOKit/storage/IOCDTypes.h>
60	#include <IOKit/storage/IODVDMedia.h>
61	#include <CoreFoundation/CoreFoundation.h>
62	#endif
63
64	#ifdef __sun__
65	#define _POSIX_PTHREAD_SEMANTICS 1
66	#include <sys/dkio.h>
67	#endif
68	#ifdef __linux__
69	#include <sys/param.h>
70	#include <sys/syscall.h>
71	#include <sys/vfs.h>
72	#if defined(CONFIG_BLKZONED)
73	#include <linux/blkzoned.h>
74	#endif
75	#include <linux/cdrom.h>
76	#include <linux/dm-ioctl.h>
77	#include <linux/fd.h>
78	#include <linux/fs.h>
79	#include <linux/hdreg.h>
80	#include <linux/magic.h>
81	#include <scsi/sg.h>
82	#ifdef __s390x__
83	#include <asm/dasd.h>
84	#endif
85	#ifndef FS_NOCOW_FL
86	#define FS_NOCOW_FL 0x00800000 /* Do not cow file */
87	#endif
88	#endif
89	#if defined(CONFIG_FALLOCATE_PUNCH_HOLE) \|\| defined(CONFIG_FALLOCATE_ZERO_RANGE)
90	#include <linux/falloc.h>
91	#endif
92	#if defined (__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
93	#include <sys/disk.h>
94	#include <sys/cdio.h>
95	#endif
96
97	#ifdef __OpenBSD__
98	#include <sys/disklabel.h>
99	#include <sys/dkio.h>
100	#endif
101
102	#ifdef __NetBSD__
103	#include <sys/disklabel.h>
104	#include <sys/dkio.h>
105	#include <sys/disk.h>
106	#endif
107
108	#ifdef __DragonFly__
109	#include <sys/diskslice.h>
110	#endif
111
112	/* OS X does not have O_DSYNC */
113	#ifndef O_DSYNC
114	#ifdef O_SYNC
115	#define O_DSYNC O_SYNC
116	#elif defined(O_FSYNC)
117	#define O_DSYNC O_FSYNC
118	#endif
119	#endif
120
121	/* Approximate O_DIRECT with O_DSYNC if O_DIRECT isn't available */
122	#ifndef O_DIRECT
123	#define O_DIRECT O_DSYNC
124	#endif
125
126	#define FTYPE_FILE 0
127	#define FTYPE_CD 1
128
129	#define MAX_BLOCKSIZE 4096
130
131	/* Posix file locking bytes. Libvirt takes byte 0, we start from higher bytes,
132	* leaving a few more bytes for its future use. */
133	#define RAW_LOCK_PERM_BASE 100
134	#define RAW_LOCK_SHARED_BASE 200
135
136	/*
137	* Multiple retries are mostly meant for two separate scenarios:
138	*
139	* - DM_MPATH_PROBE_PATHS returns success, but before SG_IO completes, another
140	* path goes down.
141	*
142	* - DM_MPATH_PROBE_PATHS failed all paths in the current path group, so we have
143	* to send another SG_IO to switch to another path group to probe the paths in
144	* it.
145	*
146	* Even if each path is in a separate path group (path_grouping_policy set to
147	* failover), it's rare to have more than eight path groups - and even then
148	* pretty unlikely that only bad path groups would be chosen in eight retries.
149	*/
150	#define SG_IO_MAX_RETRIES 8
151
152	typedef struct BDRVRawState {
153	int fd;
154	bool use_lock;
155	int type;
156	int open_flags;
157	size_t buf_align;
158
159	/* The current permissions. */
160	uint64_t perm;
161	uint64_t shared_perm;
162
163	/* The perms bits whose corresponding bytes are already locked in
164	* s->fd. */
165	uint64_t locked_perm;
166	uint64_t locked_shared_perm;
167
168	uint64_t aio_max_batch;
169
170	int perm_change_fd;
171	int perm_change_flags;
172	BDRVReopenState *reopen_state;
173
174	bool has_discard:1;
175	bool has_write_zeroes:1;
176	bool use_linux_aio:1;
177	bool has_laio_fdsync:1;
178	bool use_linux_io_uring:1;
179	bool use_mpath:1;
180	int page_cache_inconsistent; /* errno from fdatasync failure */
181	bool has_fallocate;
182	bool needs_alignment;
183	bool force_alignment;
184	bool drop_cache;
185	bool check_cache_dropped;
186	struct {
187	uint64_t discard_nb_ok;
188	uint64_t discard_nb_failed;
189	uint64_t discard_bytes_ok;
190	} stats;
191
192	PRManager *pr_mgr;
193	} BDRVRawState;
194
195	typedef struct BDRVRawReopenState {
196	int open_flags;
197	bool drop_cache;
198	bool check_cache_dropped;
199	} BDRVRawReopenState;
200
201	static int fd_open(BlockDriverState *bs)
202	{
203	BDRVRawState *s = bs->opaque;
204
205	/* this is just to ensure s->fd is sane (its called by io ops) */
206	if (s->fd >= 0) {
207	return 0;
208	}
209	return -EIO;
210	}
211
212	static int64_t raw_getlength(BlockDriverState *bs);
213	static int coroutine_fn raw_co_flush_to_disk(BlockDriverState *bs);
214
215	typedef struct RawPosixAIOData {
216	BlockDriverState *bs;
217	int aio_type;
218	int aio_fildes;
219
220	off_t aio_offset;
221	uint64_t aio_nbytes;
222
223	union {
224	struct {
225	struct iovec *iov;
226	int niov;
227	} io;
228	struct {
229	uint64_t cmd;
230	void *buf;
231	} ioctl;
232	struct {
233	int aio_fd2;
234	off_t aio_offset2;
235	} copy_range;
236	struct {
237	PreallocMode prealloc;
238	Error **errp;
239	} truncate;
240	struct {
241	unsigned int *nr_zones;
242	BlockZoneDescriptor *zones;
243	} zone_report;
244	struct {
245	unsigned long op;
246	} zone_mgmt;
247	};
248	} RawPosixAIOData;
249
250	#if defined(__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
251	static int cdrom_reopen(BlockDriverState *bs);
252	#endif
253
254	/*
255	* Elide EAGAIN and EACCES details when failing to lock, as this
256	* indicates that the specified file region is already locked by
257	* another process, which is considered a common scenario.
258	*/
259	#define raw_lock_error_setg_errno(errp, err, fmt, ...) \
260	do { \
261	if ((err) == EAGAIN \|\| (err) == EACCES) { \
262	error_setg((errp), (fmt), ## __VA_ARGS__); \
263	} else { \
264	error_setg_errno((errp), (err), (fmt), ## __VA_ARGS__); \
265	} \
266	} while (0)
267
268	#if defined(__NetBSD__)
269	static int raw_normalize_devicepath(const char filename, Error errp)
270	{
271	static char namebuf[PATH_MAX];
272	const char dp, fname;
273	struct stat sb;
274
275	fname = *filename;
276	dp = strrchr(fname, '/');
277	if (lstat(fname, &sb) < 0) {
278	error_setg_file_open(errp, errno, fname);
279	return -errno;
280	}
281
282	if (!S_ISBLK(sb.st_mode)) {
283	return 0;
284	}
285
286	if (dp == NULL) {
287	snprintf(namebuf, PATH_MAX, "r%s", fname);
288	} else {
289	snprintf(namebuf, PATH_MAX, "%.*s/r%s",
290	(int)(dp - fname), fname, dp + 1);
291	}
292	*filename = namebuf;
293	warn_report("%s is a block device, using %s", fname, *filename);
294
295	return 0;
296	}
297	#else
298	static int raw_normalize_devicepath(const char filename, Error errp)
299	{
300	return 0;
301	}
302	#endif
303
304	/*
305	* Get logical block size via ioctl. On success store it in @sector_size_p.
306	*/
307	static int probe_logical_blocksize(int fd, unsigned int *sector_size_p)
308	{
309	unsigned int sector_size;
310	bool success = false;
311	int i;
312
313	errno = ENOTSUP;
314	static const unsigned long ioctl_list[] = {
315	#ifdef BLKSSZGET
316	BLKSSZGET,
317	#endif
318	#ifdef DKIOCGETBLOCKSIZE
319	DKIOCGETBLOCKSIZE,
320	#endif
321	#ifdef DIOCGSECTORSIZE
322	DIOCGSECTORSIZE,
323	#endif
324	};
325
326	/* Try a few ioctls to get the right size */
327	for (i = 0; i < (int)ARRAY_SIZE(ioctl_list); i++) {
328	if (ioctl(fd, ioctl_list[i], &sector_size) >= 0) {
329	*sector_size_p = sector_size;
330	success = true;
331	}
332	}
333
334	return success ? 0 : -errno;
335	}
336
337	/**
338	* Get physical block size of @fd.
339	* On success, store it in @blk_size and return 0.
340	* On failure, return -errno.
341	*/
342	static int probe_physical_blocksize(int fd, unsigned int *blk_size)
343	{
344	#ifdef BLKPBSZGET
345	if (ioctl(fd, BLKPBSZGET, blk_size) < 0) {
346	return -errno;
347	}
348	return 0;
349	#else
350	return -ENOTSUP;
351	#endif
352	}
353
354	/*
355	* Returns true if no alignment restrictions are necessary even for files
356	* opened with O_DIRECT.
357	*
358	* raw_probe_alignment() probes the required alignment and assume that 1 means
359	* the probing failed, so it falls back to a safe default of 4k. This can be
360	* avoided if we know that byte alignment is okay for the file.
361	*/
362	static bool dio_byte_aligned(int fd)
363	{
364	#ifdef __linux__
365	struct statfs buf;
366	int ret;
367
368	ret = fstatfs(fd, &buf);
369	if (ret == 0 && buf.f_type == NFS_SUPER_MAGIC) {
370	return true;
371	}
372	#endif
373	return false;
374	}
375
376	static bool raw_needs_alignment(BlockDriverState *bs)
377	{
378	BDRVRawState *s = bs->opaque;
379
380	if ((bs->open_flags & BDRV_O_NOCACHE) != 0 && !dio_byte_aligned(s->fd)) {
381	return true;
382	}
383
384	return s->force_alignment;
385	}
386
387	/* Check if read is allowed with given memory buffer and length.
388	*
389	* This function is used to check O_DIRECT memory buffer and request alignment.
390	*/
391	static bool raw_is_io_aligned(int fd, void *buf, size_t len)
392	{
393	ssize_t ret = pread(fd, buf, len, 0);
394
395	if (ret >= 0) {
396	return true;
397	}
398
399	#ifdef __linux__
400	/* The Linux kernel returns EINVAL for misaligned O_DIRECT reads. Ignore
401	* other errors (e.g. real I/O error), which could happen on a failed
402	* drive, since we only care about probing alignment.
403	*/
404	if (errno != EINVAL) {
405	return true;
406	}
407	#endif
408
409	return false;
410	}
411
412	static void raw_probe_alignment(BlockDriverState bs, int fd, Error *errp)
413	{
414	BDRVRawState *s = bs->opaque;
415	char *buf;
416	size_t max_align = MAX(MAX_BLOCKSIZE, qemu_real_host_page_size());
417	size_t alignments[] = {1, 512, 1024, 2048, 4096};
418
419	/* For SCSI generic devices the alignment is not really used.
420	With buffered I/O, we don't have any restrictions. */
421	if (bdrv_is_sg(bs) \|\| !s->needs_alignment) {
422	bs->bl.request_alignment = 1;
423	s->buf_align = 1;
424	return;
425	}
426
427	bs->bl.request_alignment = 0;
428	s->buf_align = 0;
429	/* Let's try to use the logical blocksize for the alignment. */
430	if (probe_logical_blocksize(fd, &bs->bl.request_alignment) < 0) {
431	bs->bl.request_alignment = 0;
432	}
433
434	#ifdef __linux__
435	/*
436	* The XFS ioctl definitions are shipped in extra packages that might
437	* not always be available. Since we just need the XFS_IOC_DIOINFO ioctl
438	* here, we simply use our own definition instead:
439	*/
440	struct xfs_dioattr {
441	uint32_t d_mem;
442	uint32_t d_miniosz;
443	uint32_t d_maxiosz;
444	} da;
445	if (ioctl(fd, _IOR('X', 30, struct xfs_dioattr), &da) >= 0) {
446	bs->bl.request_alignment = da.d_miniosz;
447	/* The kernel returns wrong information for d_mem */
448	/* s->buf_align = da.d_mem; */
449	}
450	#endif
451
452	/*
453	* If we could not get the sizes so far, we can only guess them. First try
454	* to detect request alignment, since it is more likely to succeed. Then
455	* try to detect buf_align, which cannot be detected in some cases (e.g.
456	* Gluster). If buf_align cannot be detected, we fallback to the value of
457	* request_alignment.
458	*/
459
460	if (!bs->bl.request_alignment) {
461	int i;
462	size_t align;
463	buf = qemu_memalign(max_align, max_align);
464	for (i = 0; i < ARRAY_SIZE(alignments); i++) {
465	align = alignments[i];
466	if (raw_is_io_aligned(fd, buf, align)) {
467	/* Fallback to safe value. */
468	bs->bl.request_alignment = (align != 1) ? align : max_align;
469	break;
470	}
471	}
472	qemu_vfree(buf);
473	}
474
475	if (!s->buf_align) {
476	int i;
477	size_t align;
478	buf = qemu_memalign(max_align, 2 * max_align);
479	for (i = 0; i < ARRAY_SIZE(alignments); i++) {
480	align = alignments[i];
481	if (raw_is_io_aligned(fd, buf + align, max_align)) {
482	/* Fallback to request_alignment. */
483	s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
484	break;
485	}
486	}
487	qemu_vfree(buf);
488	}
489
490	if (!s->buf_align \|\| !bs->bl.request_alignment) {
491	error_setg(errp, "Could not find working O_DIRECT alignment");
492	error_append_hint(errp, "Try cache.direct=off\n");
493	}
494	}
495
496	static int check_hdev_writable(int fd)
497	{
498	#if defined(BLKROGET)
499	/* Linux block devices can be configured "read-only" using blockdev(8).
500	* This is independent of device node permissions and therefore open(2)
501	* with O_RDWR succeeds. Actual writes fail with EPERM.
502	*
503	* bdrv_open() is supposed to fail if the disk is read-only. Explicitly
504	* check for read-only block devices so that Linux block devices behave
505	* properly.
506	*/
507	struct stat st;
508	int readonly = 0;
509
510	if (fstat(fd, &st)) {
511	return -errno;
512	}
513
514	if (!S_ISBLK(st.st_mode)) {
515	return 0;
516	}
517
518	if (ioctl(fd, BLKROGET, &readonly) < 0) {
519	return -errno;
520	}
521
522	if (readonly) {
523	return -EACCES;
524	}
525	#endif /* defined(BLKROGET) */
526	return 0;
527	}
528
529	static void raw_parse_flags(int bdrv_flags, int *open_flags, bool has_writers)
530	{
531	bool read_write = false;
532	assert(open_flags != NULL);
533
534	*open_flags \|= O_BINARY;
535	*open_flags &= ~O_ACCMODE;
536
537	if (bdrv_flags & BDRV_O_AUTO_RDONLY) {
538	read_write = has_writers;
539	} else if (bdrv_flags & BDRV_O_RDWR) {
540	read_write = true;
541	}
542
543	if (read_write) {
544	*open_flags \|= O_RDWR;
545	} else {
546	*open_flags \|= O_RDONLY;
547	}
548
549	/* Use O_DSYNC for write-through caching, no flags for write-back caching,
550	* and O_DIRECT for no caching. */
551	if ((bdrv_flags & BDRV_O_NOCACHE)) {
552	*open_flags \|= O_DIRECT;
553	}
554	}
555
556	static void raw_parse_filename(const char filename, QDict options,
557	Error **errp)
558	{
559	bdrv_parse_filename_strip_prefix(filename, "file:", options);
560	}
561
562	static QemuOptsList raw_runtime_opts = {
563	.name = "raw",
564	.head = QTAILQ_HEAD_INITIALIZER(raw_runtime_opts.head),
565	.desc = {
566	{
567	.name = "filename",
568	.type = QEMU_OPT_STRING,
569	.help = "File name of the image",
570	},
571	{
572	.name = "aio",
573	.type = QEMU_OPT_STRING,
574	.help = "host AIO implementation (threads, native, io_uring)",
575	},
576	{
577	.name = "aio-max-batch",
578	.type = QEMU_OPT_NUMBER,
579	.help = "AIO max batch size (0 = auto handled by AIO backend, default: 0)",
580	},
581	{
582	.name = "locking",
583	.type = QEMU_OPT_STRING,
584	.help = "file locking mode (on/off/auto, default: auto)",
585	},
586	{
587	.name = "pr-manager",
588	.type = QEMU_OPT_STRING,
589	.help = "id of persistent reservation manager object (default: none)",
590	},
591	#if defined(__linux__)
592	{
593	.name = "drop-cache",
594	.type = QEMU_OPT_BOOL,
595	.help = "invalidate page cache during live migration (default: on)",
596	},
597	#endif
598	{
599	.name = "x-check-cache-dropped",
600	.type = QEMU_OPT_BOOL,
601	.help = "check that page cache was dropped on live migration (default: off)"
602	},
603	{ /* end of list */ }
604	},
605	};
606
607	static const char *const mutable_opts[] = { "x-check-cache-dropped", NULL };
608
609	static int raw_open_common(BlockDriverState bs, QDict options,
610	int bdrv_flags, int open_flags,
611	bool device, Error **errp)
612	{
613	BDRVRawState *s = bs->opaque;
614	QemuOpts *opts;
615	Error *local_err = NULL;
616	const char *filename = NULL;
617	const char *str;
618	BlockdevAioOptions aio, aio_default;
619	int fd, ret;
620	struct stat st;
621	OnOffAuto locking;
622
623	opts = qemu_opts_create(&raw_runtime_opts, NULL, 0, &error_abort);
624	if (!qemu_opts_absorb_qdict(opts, options, errp)) {
625	ret = -EINVAL;
626	goto fail;
627	}
628
629	filename = qemu_opt_get(opts, "filename");
630
631	ret = raw_normalize_devicepath(&filename, errp);
632	if (ret != 0) {
633	goto fail;
634	}
635
636	if (bdrv_flags & BDRV_O_NATIVE_AIO) {
637	aio_default = BLOCKDEV_AIO_OPTIONS_NATIVE;
638	#ifdef CONFIG_LINUX_IO_URING
639	} else if (bdrv_flags & BDRV_O_IO_URING) {
640	aio_default = BLOCKDEV_AIO_OPTIONS_IO_URING;
641	#endif
642	} else {
643	aio_default = BLOCKDEV_AIO_OPTIONS_THREADS;
644	}
645
646	aio = qapi_enum_parse(&BlockdevAioOptions_lookup,
647	qemu_opt_get(opts, "aio"),
648	aio_default, &local_err);
649	if (local_err) {
650	error_propagate(errp, local_err);
651	ret = -EINVAL;
652	goto fail;
653	}
654
655	s->use_linux_aio = (aio == BLOCKDEV_AIO_OPTIONS_NATIVE);
656	#ifdef CONFIG_LINUX_IO_URING
657	s->use_linux_io_uring = (aio == BLOCKDEV_AIO_OPTIONS_IO_URING);
658	#endif
659
660	s->aio_max_batch = qemu_opt_get_number(opts, "aio-max-batch", 0);
661
662	locking = qapi_enum_parse(&OnOffAuto_lookup,
663	qemu_opt_get(opts, "locking"),
664	ON_OFF_AUTO_AUTO, &local_err);
665	if (local_err) {
666	error_propagate(errp, local_err);
667	ret = -EINVAL;
668	goto fail;
669	}
670	switch (locking) {
671	case ON_OFF_AUTO_ON:
672	s->use_lock = true;
673	if (!qemu_has_ofd_lock()) {
674	warn_report("File lock requested but OFD locking syscall is "
675	"unavailable, falling back to POSIX file locks");
676	error_printf("Due to the implementation, locks can be lost "
677	"unexpectedly.\n");
678	}
679	break;
680	case ON_OFF_AUTO_OFF:
681	s->use_lock = false;
682	break;
683	case ON_OFF_AUTO_AUTO:
684	s->use_lock = qemu_has_ofd_lock();
685	break;
686	default:
687	abort();
688	}
689
690	str = qemu_opt_get(opts, "pr-manager");
691	if (str) {
692	s->pr_mgr = pr_manager_lookup(str, &local_err);
693	if (local_err) {
694	error_propagate(errp, local_err);
695	ret = -EINVAL;
696	goto fail;
697	}
698	}
699
700	s->drop_cache = qemu_opt_get_bool(opts, "drop-cache", true);
701	s->check_cache_dropped = qemu_opt_get_bool(opts, "x-check-cache-dropped",
702	false);
703
704	s->open_flags = open_flags;
705	raw_parse_flags(bdrv_flags, &s->open_flags, false);
706
707	s->fd = -1;
708	fd = qemu_open(filename, s->open_flags, errp);
709	ret = fd < 0 ? -errno : 0;
710
711	if (ret < 0) {
712	if (ret == -EROFS) {
713	ret = -EACCES;
714	}
715	goto fail;
716	}
717	s->fd = fd;
718
719	/* Check s->open_flags rather than bdrv_flags due to auto-read-only */
720	if (s->open_flags & O_RDWR) {
721	ret = check_hdev_writable(s->fd);
722	if (ret < 0) {
723	error_setg_errno(errp, -ret, "The device is not writable");
724	goto fail;
725	}
726	}
727
728	s->perm = 0;
729	s->shared_perm = BLK_PERM_ALL;
730
731	#ifdef CONFIG_LINUX_AIO
732	/* Currently Linux does AIO only for files opened with O_DIRECT */
733	if (s->use_linux_aio && !(s->open_flags & O_DIRECT)) {
734	error_setg(errp, "aio=native was specified, but it requires "
735	"cache.direct=on, which was not specified.");
736	ret = -EINVAL;
737	goto fail;
738	}
739	if (s->use_linux_aio) {
740	s->has_laio_fdsync = laio_has_fdsync(s->fd);
741	}
742	#else
743	if (s->use_linux_aio) {
744	error_setg(errp, "aio=native was specified, but is not supported "
745	"in this build.");
746	ret = -EINVAL;
747	goto fail;
748	}
749	#endif /* !defined(CONFIG_LINUX_AIO) */
750
751	if (s->use_linux_io_uring) {
752	#ifdef CONFIG_LINUX_IO_URING
753	if (!aio_has_io_uring()) {
754	error_setg(errp, "aio=io_uring was specified, but is not "
755	"available (disabled via io_uring_disabled "
756	"sysctl or blocked by container runtime "
757	"seccomp policy?)");
758	ret = -EINVAL;
759	goto fail;
760	}
761	#else
762	error_setg(errp, "aio=io_uring was specified, but is not supported "
763	"in this build");
764	ret = -EINVAL;
765	goto fail;
766	#endif /* !defined(CONFIG_LINUX_IO_URING) */
767	}
768
769	s->has_discard = true;
770	s->has_write_zeroes = true;
771
772	if (fstat(s->fd, &st) < 0) {
773	ret = -errno;
774	error_setg_errno(errp, errno, "Could not stat file");
775	goto fail;
776	}
777
778	if (!device) {
779	if (!S_ISREG(st.st_mode)) {
780	error_setg(errp, "'%s' driver requires '%s' to be a regular file",
781	bs->drv->format_name, bs->filename);
782	ret = -EINVAL;
783	goto fail;
784	} else {
785	s->has_fallocate = true;
786	}
787	} else {
788	if (!(S_ISCHR(st.st_mode) \|\| S_ISBLK(st.st_mode))) {
789	error_setg(errp, "'%s' driver requires '%s' to be either "
790	"a character or block device",
791	bs->drv->format_name, bs->filename);
792	ret = -EINVAL;
793	goto fail;
794	}
795	}
796
797	#ifdef __FreeBSD__
798	if (S_ISCHR(st.st_mode)) {
799	/*
800	* The file is a char device (disk), which on FreeBSD isn't behind
801	* a pager, so force all requests to be aligned. This is needed
802	* so QEMU makes sure all IO operations on the device are aligned
803	* to sector size, or else FreeBSD will reject them with EINVAL.
804	*/
805	s->force_alignment = true;
806	}
807	#endif
808	s->needs_alignment = raw_needs_alignment(bs);
809
810	bs->supported_write_flags = BDRV_REQ_FUA;
811	if (s->use_linux_aio && !laio_has_fua()) {
812	bs->supported_write_flags &= ~BDRV_REQ_FUA;
813	} else if (s->use_linux_io_uring && !luring_has_fua()) {
814	bs->supported_write_flags &= ~BDRV_REQ_FUA;
815	}
816
817	bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP \| BDRV_REQ_NO_FALLBACK;
818	if (S_ISREG(st.st_mode)) {
819	/* When extending regular files, we get zeros from the OS */
820	bs->supported_truncate_flags = BDRV_REQ_ZERO_WRITE;
821	}
822	ret = 0;
823	fail:
824	if (ret < 0 && s->fd != -1) {
825	qemu_close(s->fd);
826	}
827	if (filename && (bdrv_flags & BDRV_O_TEMPORARY)) {
828	unlink(filename);
829	}
830	qemu_opts_del(opts);
831	return ret;
832	}
833
834	static int raw_open(BlockDriverState bs, QDict options, int flags,
835	Error **errp)
836	{
837	BDRVRawState *s = bs->opaque;
838
839	s->type = FTYPE_FILE;
840	return raw_open_common(bs, options, flags, 0, false, errp);
841	}
842
843	typedef enum {
844	RAW_PL_PREPARE,
845	RAW_PL_COMMIT,
846	RAW_PL_ABORT,
847	} RawPermLockOp;
848
849	#define PERM_FOREACH(i) \
850	for ((i) = 0; (1ULL << (i)) <= BLK_PERM_ALL; i++)
851
852	/* Lock bytes indicated by @perm_lock_bits and @shared_perm_lock_bits in the
853	* file; if @unlock == true, also unlock the unneeded bytes.
854	* @shared_perm_lock_bits is the mask of all permissions that are NOT shared.
855	*/
856	static int raw_apply_lock_bytes(BDRVRawState *s, int fd,
857	uint64_t perm_lock_bits,
858	uint64_t shared_perm_lock_bits,
859	bool unlock, Error **errp)
860	{
861	int ret;
862	int i;
863	uint64_t locked_perm, locked_shared_perm;
864
865	if (s) {
866	locked_perm = s->locked_perm;
867	locked_shared_perm = s->locked_shared_perm;
868	} else {
869	/*
870	* We don't have the previous bits, just lock/unlock for each of the
871	* requested bits.
872	*/
873	if (unlock) {
874	locked_perm = BLK_PERM_ALL;
875	locked_shared_perm = BLK_PERM_ALL;
876	} else {
877	locked_perm = 0;
878	locked_shared_perm = 0;
879	}
880	}
881
882	PERM_FOREACH(i) {
883	int off = RAW_LOCK_PERM_BASE + i;
884	uint64_t bit = (1ULL << i);
885	if ((perm_lock_bits & bit) && !(locked_perm & bit)) {
886	ret = qemu_lock_fd(fd, off, 1, false);
887	if (ret) {
888	raw_lock_error_setg_errno(errp, -ret, "Failed to lock byte %d",
889	off);
890	return ret;
891	} else if (s) {
892	s->locked_perm \|= bit;
893	}
894	} else if (unlock && (locked_perm & bit) && !(perm_lock_bits & bit)) {
895	ret = qemu_unlock_fd(fd, off, 1);
896	if (ret) {
897	error_setg_errno(errp, -ret, "Failed to unlock byte %d", off);
898	return ret;
899	} else if (s) {
900	s->locked_perm &= ~bit;
901	}
902	}
903	}
904	PERM_FOREACH(i) {
905	int off = RAW_LOCK_SHARED_BASE + i;
906	uint64_t bit = (1ULL << i);
907	if ((shared_perm_lock_bits & bit) && !(locked_shared_perm & bit)) {
908	ret = qemu_lock_fd(fd, off, 1, false);
909	if (ret) {
910	raw_lock_error_setg_errno(errp, -ret, "Failed to lock byte %d",
911	off);
912	return ret;
913	} else if (s) {
914	s->locked_shared_perm \|= bit;
915	}
916	} else if (unlock && (locked_shared_perm & bit) &&
917	!(shared_perm_lock_bits & bit)) {
918	ret = qemu_unlock_fd(fd, off, 1);
919	if (ret) {
920	error_setg_errno(errp, -ret, "Failed to unlock byte %d", off);
921	return ret;
922	} else if (s) {
923	s->locked_shared_perm &= ~bit;
924	}
925	}
926	}
927	return 0;
928	}
929
930	/* Check "unshared" bytes implied by @perm and ~@shared_perm in the file. */
931	static int raw_check_lock_bytes(int fd, uint64_t perm, uint64_t shared_perm,
932	Error **errp)
933	{
934	int ret;
935	int i;
936
937	PERM_FOREACH(i) {
938	int off = RAW_LOCK_SHARED_BASE + i;
939	uint64_t p = 1ULL << i;
940	if (perm & p) {
941	ret = qemu_lock_fd_test(fd, off, 1, true);
942	if (ret) {
943	char *perm_name = bdrv_perm_names(p);
944
945	raw_lock_error_setg_errno(errp, -ret,
946	"Failed to get \"%s\" lock",
947	perm_name);
948	g_free(perm_name);
949	return ret;
950	}
951	}
952	}
953	PERM_FOREACH(i) {
954	int off = RAW_LOCK_PERM_BASE + i;
955	uint64_t p = 1ULL << i;
956	if (!(shared_perm & p)) {
957	ret = qemu_lock_fd_test(fd, off, 1, true);
958	if (ret) {
959	char *perm_name = bdrv_perm_names(p);
960
961	raw_lock_error_setg_errno(errp, -ret,
962	"Failed to get shared \"%s\" lock",
963	perm_name);
964	g_free(perm_name);
965	return ret;
966	}
967	}
968	}
969	return 0;
970	}
971
972	static int raw_handle_perm_lock(BlockDriverState *bs,
973	RawPermLockOp op,
974	uint64_t new_perm, uint64_t new_shared,
975	Error **errp)
976	{
977	BDRVRawState *s = bs->opaque;
978	int ret = 0;
979	Error *local_err = NULL;
980
981	if (!s->use_lock) {
982	return 0;
983	}
984
985	if (bdrv_get_flags(bs) & BDRV_O_INACTIVE) {
986	return 0;
987	}
988
989	switch (op) {
990	case RAW_PL_PREPARE:
991	if ((s->perm \| new_perm) == s->perm &&
992	(s->shared_perm & new_shared) == s->shared_perm)
993	{
994	/*
995	* We are going to unlock bytes, it should not fail. If it fail due
996	* to some fs-dependent permission-unrelated reasons (which occurs
997	* sometimes on NFS and leads to abort in bdrv_replace_child) we
998	* can't prevent such errors by any check here. And we ignore them
999	* anyway in ABORT and COMMIT.
1000	*/
1001	return 0;
1002	}
1003	ret = raw_apply_lock_bytes(s, s->fd, s->perm \| new_perm,
1004	~s->shared_perm \| ~new_shared,
1005	false, errp);
1006	if (!ret) {
1007	ret = raw_check_lock_bytes(s->fd, new_perm, new_shared, errp);
1008	if (!ret) {
1009	return 0;
1010	}
1011	error_append_hint(errp,
1012	"Is another process using the image [%s]?\n",
1013	bs->filename);
1014	}
1015	/* fall through to unlock bytes. */
1016	case RAW_PL_ABORT:
1017	raw_apply_lock_bytes(s, s->fd, s->perm, ~s->shared_perm,
1018	true, &local_err);
1019	if (local_err) {
1020	/* Theoretically the above call only unlocks bytes and it cannot
1021	* fail. Something weird happened, report it.
1022	*/
1023	warn_report_err(local_err);
1024	}
1025	break;
1026	case RAW_PL_COMMIT:
1027	raw_apply_lock_bytes(s, s->fd, new_perm, ~new_shared,
1028	true, &local_err);
1029	if (local_err) {
1030	/* Theoretically the above call only unlocks bytes and it cannot
1031	* fail. Something weird happened, report it.
1032	*/
1033	warn_report_err(local_err);
1034	}
1035	break;
1036	}
1037	return ret;
1038	}
1039
1040	static int raw_reconfigure_getfd(BlockDriverState *bs, int flags,
1041	int open_flags, uint64_t perm, Error *errp)
1042	{
1043	BDRVRawState *s = bs->opaque;
1044	int fd = -1;
1045	int ret;
1046	bool has_writers = perm &
1047	(BLK_PERM_WRITE \| BLK_PERM_WRITE_UNCHANGED \| BLK_PERM_RESIZE);
1048	int fcntl_flags = O_APPEND \| O_NONBLOCK;
1049	#ifdef O_NOATIME
1050	fcntl_flags \|= O_NOATIME;
1051	#endif
1052
1053	*open_flags = 0;
1054	if (s->type == FTYPE_CD) {
1055	*open_flags \|= O_NONBLOCK;
1056	}
1057
1058	raw_parse_flags(flags, open_flags, has_writers);
1059
1060	#ifdef O_ASYNC
1061	/* Not all operating systems have O_ASYNC, and those that don't
1062	* will not let us track the state into rs->open_flags (typically
1063	* you achieve the same effect with an ioctl, for example I_SETSIG
1064	* on Solaris). But we do not use O_ASYNC, so that's fine.
1065	*/
1066	assert((s->open_flags & O_ASYNC) == 0);
1067	#endif
1068
1069	if (*open_flags == s->open_flags) {
1070	/* We're lucky, the existing fd is fine */
1071	return s->fd;
1072	}
1073
1074	if ((*open_flags & ~fcntl_flags) == (s->open_flags & ~fcntl_flags)) {
1075	/* dup the original fd */
1076	fd = qemu_dup(s->fd);
1077	if (fd >= 0) {
1078	ret = qemu_fcntl_addfl(fd, *open_flags);
1079	if (ret) {
1080	qemu_close(fd);
1081	fd = -1;
1082	}
1083	}
1084	}
1085
1086	/* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
1087	if (fd == -1) {
1088	const char *normalized_filename = bs->filename;
1089	ret = raw_normalize_devicepath(&normalized_filename, errp);
1090	if (ret >= 0) {
1091	fd = qemu_open(normalized_filename, *open_flags, errp);
1092	if (fd == -1) {
1093	return -1;
1094	}
1095	}
1096	}
1097
1098	if (fd != -1 && (*open_flags & O_RDWR)) {
1099	ret = check_hdev_writable(fd);
1100	if (ret < 0) {
1101	qemu_close(fd);
1102	error_setg_errno(errp, -ret, "The device is not writable");
1103	return -1;
1104	}
1105	}
1106
1107	return fd;
1108	}
1109
1110	static int raw_reopen_prepare(BDRVReopenState *state,
1111	BlockReopenQueue queue, Error *errp)
1112	{
1113	BDRVRawState *s;
1114	BDRVRawReopenState *rs;
1115	QemuOpts *opts;
1116	int ret;
1117
1118	assert(state != NULL);
1119	assert(state->bs != NULL);
1120
1121	s = state->bs->opaque;
1122
1123	state->opaque = g_new0(BDRVRawReopenState, 1);
1124	rs = state->opaque;
1125
1126	/* Handle options changes */
1127	opts = qemu_opts_create(&raw_runtime_opts, NULL, 0, &error_abort);
1128	if (!qemu_opts_absorb_qdict(opts, state->options, errp)) {
1129	ret = -EINVAL;
1130	goto out;
1131	}
1132
1133	rs->drop_cache = qemu_opt_get_bool_del(opts, "drop-cache", true);
1134	rs->check_cache_dropped =
1135	qemu_opt_get_bool_del(opts, "x-check-cache-dropped", false);
1136
1137	/* This driver's reopen function doesn't currently allow changing
1138	* other options, so let's put them back in the original QDict and
1139	* bdrv_reopen_prepare() will detect changes and complain. */
1140	qemu_opts_to_qdict(opts, state->options);
1141
1142	/*
1143	* As part of reopen prepare we also want to create new fd by
1144	* raw_reconfigure_getfd(). But it wants updated "perm", when in
1145	* bdrv_reopen_multiple() .bdrv_reopen_prepare() callback called prior to
1146	* permission update. Happily, permission update is always a part
1147	* (a separate stage) of bdrv_reopen_multiple() so we can rely on this
1148	* fact and reconfigure fd in raw_check_perm().
1149	*/
1150
1151	s->reopen_state = state;
1152	ret = 0;
1153
1154	out:
1155	qemu_opts_del(opts);
1156	return ret;
1157	}
1158
1159	static void raw_reopen_commit(BDRVReopenState *state)
1160	{
1161	BDRVRawReopenState *rs = state->opaque;
1162	BDRVRawState *s = state->bs->opaque;
1163
1164	s->drop_cache = rs->drop_cache;
1165	s->check_cache_dropped = rs->check_cache_dropped;
1166	s->open_flags = rs->open_flags;
1167	g_free(state->opaque);
1168	state->opaque = NULL;
1169
1170	assert(s->reopen_state == state);
1171	s->reopen_state = NULL;
1172	}
1173
1174
1175	static void raw_reopen_abort(BDRVReopenState *state)
1176	{
1177	BDRVRawReopenState *rs = state->opaque;
1178	BDRVRawState *s = state->bs->opaque;
1179
1180	/* nothing to do if NULL, we didn't get far enough */
1181	if (rs == NULL) {
1182	return;
1183	}
1184
1185	g_free(state->opaque);
1186	state->opaque = NULL;
1187
1188	assert(s->reopen_state == state);
1189	s->reopen_state = NULL;
1190	}
1191
1192	static int hdev_get_max_hw_transfer(int fd, struct stat *st)
1193	{
1194	#ifdef BLKSECTGET
1195	if (S_ISBLK(st->st_mode)) {
1196	unsigned short max_sectors = 0;
1197	if (ioctl(fd, BLKSECTGET, &max_sectors) == 0) {
1198	return max_sectors * 512;
1199	}
1200	} else {
1201	int max_bytes = 0;
1202	if (ioctl(fd, BLKSECTGET, &max_bytes) == 0) {
1203	return max_bytes;
1204	}
1205	}
1206	return -errno;
1207	#else
1208	return -ENOSYS;
1209	#endif
1210	}
1211
1212	/*
1213	* Get a sysfs attribute value as character string.
1214	*/
1215	#ifdef CONFIG_LINUX
1216	static int get_sysfs_str_val(struct stat st, const char attribute,
1217	char **val) {
1218	g_autofree char *sysfspath = NULL;
1219	size_t len;
1220
1221	if (!S_ISBLK(st->st_mode)) {
1222	return -ENOTSUP;
1223	}
1224
1225	sysfspath = g_strdup_printf("/sys/dev/block/%u:%u/queue/%s",
1226	major(st->st_rdev), minor(st->st_rdev),
1227	attribute);
1228	if (!g_file_get_contents(sysfspath, val, &len, NULL)) {
1229	return -ENOENT;
1230	}
1231
1232	/* The file is ended with '\n' */
1233	char *p;
1234	p = *val;
1235	if (*(p + len - 1) == '\n') {
1236	*(p + len - 1) = '\0';
1237	}
1238	return 0;
1239	}
1240	#endif
1241
1242	#if defined(CONFIG_BLKZONED)
1243	static int get_sysfs_zoned_model(struct stat st, BlockZoneModel zoned)
1244	{
1245	g_autofree char *val = NULL;
1246	int ret;
1247
1248	ret = get_sysfs_str_val(st, "zoned", &val);
1249	if (ret < 0) {
1250	return ret;
1251	}
1252
1253	if (strcmp(val, "host-managed") == 0) {
1254	*zoned = BLK_Z_HM;
1255	} else if (strcmp(val, "host-aware") == 0) {
1256	*zoned = BLK_Z_HA;
1257	} else if (strcmp(val, "none") == 0) {
1258	*zoned = BLK_Z_NONE;
1259	} else {
1260	return -ENOTSUP;
1261	}
1262	return 0;
1263	}
1264	#endif /* defined(CONFIG_BLKZONED) */
1265
1266	#ifdef CONFIG_LINUX
1267	/*
1268	* Get a sysfs attribute value as a long integer.
1269	*/
1270	static long get_sysfs_long_val(struct stat st, const char attribute)
1271	{
1272	g_autofree char *str = NULL;
1273	const char *end;
1274	long val;
1275	int ret;
1276
1277	ret = get_sysfs_str_val(st, attribute, &str);
1278	if (ret < 0) {
1279	return ret;
1280	}
1281
1282	/* The file is ended with '\n', pass 'end' to accept that. */
1283	ret = qemu_strtol(str, &end, 10, &val);
1284	if (ret == 0 && end && *end == '\0') {
1285	ret = val;
1286	}
1287	return ret;
1288	}
1289
1290	/*
1291	* Get a sysfs attribute value as a uint32_t.
1292	*/
1293	static int get_sysfs_u32_val(struct stat st, const char attribute,
1294	uint32_t *u32)
1295	{
1296	g_autofree char *str = NULL;
1297	const char *end;
1298	unsigned int val;
1299	int ret;
1300
1301	ret = get_sysfs_str_val(st, attribute, &str);
1302	if (ret < 0) {
1303	return ret;
1304	}
1305
1306	/* The file is ended with '\n', pass 'end' to accept that. */
1307	ret = qemu_strtoui(str, &end, 10, &val);
1308	if (ret == 0 && end && *end == '\0') {
1309	*u32 = val;
1310	}
1311	return ret;
1312	}
1313	#endif
1314
1315	static int hdev_get_max_segments(int fd, struct stat *st)
1316	{
1317	#ifdef CONFIG_LINUX
1318	int ret;
1319
1320	if (S_ISCHR(st->st_mode)) {
1321	if (ioctl(fd, SG_GET_SG_TABLESIZE, &ret) == 0) {
1322	return ret;
1323	}
1324	return -ENOTSUP;
1325	}
1326	return get_sysfs_long_val(st, "max_segments");
1327	#else
1328	return -ENOTSUP;
1329	#endif
1330	}
1331
1332	/*
1333	* Fills in *dalign with the discard alignment and returns 0 on success,
1334	* -errno otherwise.
1335	*/
1336	static int hdev_get_pdiscard_alignment(struct stat st, uint32_t dalign)
1337	{
1338	#ifdef CONFIG_LINUX
1339	/*
1340	* Note that Linux "discard_granularity" is QEMU "discard_alignment". Linux
1341	* "discard_alignment" is something else.
1342	*/
1343	return get_sysfs_u32_val(st, "discard_granularity", dalign);
1344	#else
1345	return -ENOTSUP;
1346	#endif
1347	}
1348
1349	#if defined(CONFIG_BLKZONED)
1350	/*
1351	* If the reset_all flag is true, then the wps of zone whose state is
1352	* not readonly or offline should be all reset to the start sector.
1353	* Else, take the real wp of the device.
1354	*/
1355	static int get_zones_wp(BlockDriverState *bs, int fd, int64_t offset,
1356	unsigned int nrz, bool reset_all)
1357	{
1358	struct blk_zone *blkz;
1359	size_t rep_size;
1360	uint64_t sector = offset >> BDRV_SECTOR_BITS;
1361	BlockZoneWps *wps = bs->wps;
1362	unsigned int j = offset / bs->bl.zone_size;
1363	unsigned int n = 0, i = 0;
1364	int ret;
1365	rep_size = sizeof(struct blk_zone_report) + nrz * sizeof(struct blk_zone);
1366	g_autofree struct blk_zone_report *rep = NULL;
1367
1368	rep = g_malloc(rep_size);
1369	blkz = (struct blk_zone *)(rep + 1);
1370	while (n < nrz) {
1371	memset(rep, 0, rep_size);
1372	rep->sector = sector;
1373	rep->nr_zones = nrz - n;
1374
1375	do {
1376	ret = ioctl(fd, BLKREPORTZONE, rep);
1377	} while (ret != 0 && errno == EINTR);
1378	if (ret != 0) {
1379	error_report("%d: ioctl BLKREPORTZONE at %" PRId64 " failed %d",
1380	fd, offset, errno);
1381	return -errno;
1382	}
1383
1384	if (!rep->nr_zones) {
1385	break;
1386	}
1387
1388	for (i = 0; i < rep->nr_zones; ++i, ++n, ++j) {
1389	/*
1390	* The wp tracking cares only about sequential writes required and
1391	* sequential write preferred zones so that the wp can advance to
1392	* the right location.
1393	* Use the most significant bit of the wp location to indicate the
1394	* zone type: 0 for SWR/SWP zones and 1 for conventional zones.
1395	*/
1396	if (blkz[i].type == BLK_ZONE_TYPE_CONVENTIONAL) {
1397	wps->wp[j] \|= 1ULL << 63;
1398	} else {
1399	switch(blkz[i].cond) {
1400	case BLK_ZONE_COND_FULL:
1401	case BLK_ZONE_COND_READONLY:
1402	/* Zone not writable */
1403	wps->wp[j] = (blkz[i].start + blkz[i].len) << BDRV_SECTOR_BITS;
1404	break;
1405	case BLK_ZONE_COND_OFFLINE:
1406	/* Zone not writable nor readable */
1407	wps->wp[j] = (blkz[i].start) << BDRV_SECTOR_BITS;
1408	break;
1409	default:
1410	if (reset_all) {
1411	wps->wp[j] = blkz[i].start << BDRV_SECTOR_BITS;
1412	} else {
1413	wps->wp[j] = blkz[i].wp << BDRV_SECTOR_BITS;
1414	}
1415	break;
1416	}
1417	}
1418	}
1419	sector = blkz[i - 1].start + blkz[i - 1].len;
1420	}
1421
1422	return 0;
1423	}
1424
1425	static void update_zones_wp(BlockDriverState *bs, int fd, int64_t offset,
1426	unsigned int nrz)
1427	{
1428	if (get_zones_wp(bs, fd, offset, nrz, 0) < 0) {
1429	error_report("update zone wp failed");
1430	}
1431	}
1432
1433	static void raw_refresh_zoned_limits(BlockDriverState bs, struct stat st,
1434	Error **errp)
1435	{
1436	BDRVRawState *s = bs->opaque;
1437	BlockZoneModel zoned = BLK_Z_NONE;
1438	int ret;
1439
1440	ret = get_sysfs_zoned_model(st, &zoned);
1441	if (ret < 0 \|\| zoned == BLK_Z_NONE) {
1442	goto no_zoned;
1443	}
1444	bs->bl.zoned = zoned;
1445
1446	/*
1447	* The kernel page cache does not reliably work for writes to SWR zones of
1448	* zoned block devices because it can not guarantee the order of writes.
1449	*/
1450	if (!(s->open_flags & O_DIRECT)) {
1451	error_setg(errp, "The driver supports zoned devices, and it requires "
1452	"cache.direct=on, which was not specified.");
1453	goto no_zoned;
1454	}
1455
1456	ret = get_sysfs_long_val(st, "max_open_zones");
1457	if (ret >= 0) {
1458	bs->bl.max_open_zones = ret;
1459	}
1460
1461	ret = get_sysfs_long_val(st, "max_active_zones");
1462	if (ret >= 0) {
1463	bs->bl.max_active_zones = ret;
1464	}
1465
1466	/*
1467	* The zoned device must at least have zone size and nr_zones fields.
1468	*/
1469	ret = get_sysfs_long_val(st, "chunk_sectors");
1470	if (ret < 0) {
1471	error_setg_errno(errp, -ret, "Unable to read chunk_sectors "
1472	"sysfs attribute");
1473	goto no_zoned;
1474	} else if (!ret) {
1475	error_setg(errp, "Read 0 from chunk_sectors sysfs attribute");
1476	goto no_zoned;
1477	}
1478	bs->bl.zone_size = ret << BDRV_SECTOR_BITS;
1479
1480	ret = get_sysfs_long_val(st, "nr_zones");
1481	if (ret < 0) {
1482	error_setg_errno(errp, -ret, "Unable to read nr_zones "
1483	"sysfs attribute");
1484	goto no_zoned;
1485	} else if (!ret) {
1486	error_setg(errp, "Read 0 from nr_zones sysfs attribute");
1487	goto no_zoned;
1488	}
1489	bs->bl.nr_zones = ret;
1490
1491	ret = get_sysfs_long_val(st, "zone_append_max_bytes");
1492	if (ret > 0) {
1493	bs->bl.max_append_sectors = ret >> BDRV_SECTOR_BITS;
1494	}
1495
1496	ret = get_sysfs_long_val(st, "zone_write_granularity");
1497	if (ret >= 0) {
1498	bs->bl.write_granularity = ret;
1499	}
1500
1501	/* The refresh_limits() function can be called multiple times. */
1502	g_free(bs->wps);
1503	bs->wps = g_malloc(sizeof(BlockZoneWps) +
1504	sizeof(int64_t) * bs->bl.nr_zones);
1505	ret = get_zones_wp(bs, s->fd, 0, bs->bl.nr_zones, 0);
1506	if (ret < 0) {
1507	error_setg_errno(errp, -ret, "report wps failed");
1508	goto no_zoned;
1509	}
1510	qemu_co_mutex_init(&bs->wps->colock);
1511	return;
1512
1513	no_zoned:
1514	bs->bl.zoned = BLK_Z_NONE;
1515	g_free(bs->wps);
1516	bs->wps = NULL;
1517	}
1518	#else /* !defined(CONFIG_BLKZONED) */
1519	static void raw_refresh_zoned_limits(BlockDriverState bs, struct stat st,
1520	Error **errp)
1521	{
1522	bs->bl.zoned = BLK_Z_NONE;
1523	}
1524	#endif /* !defined(CONFIG_BLKZONED) */
1525
1526	static void raw_refresh_limits(BlockDriverState bs, Error *errp)
1527	{
1528	BDRVRawState *s = bs->opaque;
1529	struct stat st;
1530
1531	s->needs_alignment = raw_needs_alignment(bs);
1532	raw_probe_alignment(bs, s->fd, errp);
1533
1534	bs->bl.min_mem_alignment = s->buf_align;
1535	bs->bl.opt_mem_alignment = MAX(s->buf_align, qemu_real_host_page_size());
1536
1537	/*
1538	* Maximum transfers are best effort, so it is okay to ignore any
1539	* errors. That said, based on the man page errors in fstat would be
1540	* very much unexpected; the only possible case seems to be ENOMEM.
1541	*/
1542	if (fstat(s->fd, &st)) {
1543	return;
1544	}
1545
1546	#if defined(__APPLE__) && (__MACH__)
1547	struct statfs buf;
1548
1549	if (!fstatfs(s->fd, &buf)) {
1550	bs->bl.opt_transfer = buf.f_iosize;
1551	bs->bl.pdiscard_alignment = buf.f_bsize;
1552	}
1553	#endif
1554
1555	if (bdrv_is_sg(bs) \|\| S_ISBLK(st.st_mode)) {
1556	int ret = hdev_get_max_hw_transfer(s->fd, &st);
1557
1558	if (ret > 0 && ret <= BDRV_REQUEST_MAX_BYTES) {
1559	bs->bl.max_hw_transfer = ret;
1560	}
1561
1562	ret = hdev_get_max_segments(s->fd, &st);
1563	if (ret > 0) {
1564	bs->bl.max_hw_iov = ret;
1565	}
1566	}
1567
1568	if (S_ISBLK(st.st_mode)) {
1569	uint32_t dalign = 0;
1570	int ret;
1571
1572	ret = hdev_get_pdiscard_alignment(&st, &dalign);
1573	if (ret == 0 && dalign != 0) {
1574	uint32_t ralign = bs->bl.request_alignment;
1575
1576	/* Probably never happens, but handle it just in case */
1577	if (dalign < ralign && (ralign % dalign == 0)) {
1578	dalign = ralign;
1579	}
1580
1581	/* The block layer requires a multiple of request_alignment */
1582	if (dalign % ralign != 0) {
1583	error_setg(errp, "Invalid pdiscard_alignment limit %u is not a "
1584	"multiple of request_alignment %u", dalign, ralign);
1585	return;
1586	}
1587
1588	bs->bl.pdiscard_alignment = dalign;
1589	}
1590
1591	#ifdef __linux__
1592	/*
1593	* Linux requires logical block size alignment for write zeroes even
1594	* when normal reads/writes do not require alignment.
1595	*/
1596	if (!s->needs_alignment) {
1597	ret = probe_logical_blocksize(s->fd,
1598	&bs->bl.pwrite_zeroes_alignment);
1599	if (ret < 0) {
1600	error_setg_errno(errp, -ret,
1601	"Failed to probe logical block size");
1602	return;
1603	}
1604	}
1605	#endif /* __linux__ */
1606	}
1607
1608	raw_refresh_zoned_limits(bs, &st, errp);
1609	}
1610
1611	static int check_for_dasd(int fd)
1612	{
1613	#ifdef BIODASDINFO2
1614	struct dasd_information2_t info = {0};
1615
1616	return ioctl(fd, BIODASDINFO2, &info);
1617	#else
1618	return -1;
1619	#endif
1620	}
1621
1622	/**
1623	* Try to get @bs's logical and physical block size.
1624	* On success, store them in @bsz and return zero.
1625	* On failure, return negative errno.
1626	*/
1627	static int hdev_probe_blocksizes(BlockDriverState bs, BlockSizes bsz)
1628	{
1629	BDRVRawState *s = bs->opaque;
1630	int ret;
1631
1632	/* If DASD or zoned devices, get blocksizes */
1633	if (check_for_dasd(s->fd) < 0) {
1634	/* zoned devices are not DASD */
1635	if (bs->bl.zoned == BLK_Z_NONE) {
1636	return -ENOTSUP;
1637	}
1638	}
1639	ret = probe_logical_blocksize(s->fd, &bsz->log);
1640	if (ret < 0) {
1641	return ret;
1642	}
1643	return probe_physical_blocksize(s->fd, &bsz->phys);
1644	}
1645
1646	/**
1647	* Try to get @bs's geometry: cyls, heads, sectors.
1648	* On success, store them in @geo and return 0.
1649	* On failure return -errno.
1650	* (Allows block driver to assign default geometry values that guest sees)
1651	*/
1652	#ifdef __linux__
1653	static int hdev_probe_geometry(BlockDriverState bs, HDGeometry geo)
1654	{
1655	BDRVRawState *s = bs->opaque;
1656	struct hd_geometry ioctl_geo = {0};
1657
1658	/* If DASD, get its geometry */
1659	if (check_for_dasd(s->fd) < 0) {
1660	return -ENOTSUP;
1661	}
1662	if (ioctl(s->fd, HDIO_GETGEO, &ioctl_geo) < 0) {
1663	return -errno;
1664	}
1665	/* HDIO_GETGEO may return success even though geo contains zeros
1666	(e.g. certain multipath setups) */
1667	if (!ioctl_geo.heads \|\| !ioctl_geo.sectors \|\| !ioctl_geo.cylinders) {
1668	return -ENOTSUP;
1669	}
1670	/* Do not return a geometry for partition */
1671	if (ioctl_geo.start != 0) {
1672	return -ENOTSUP;
1673	}
1674	geo->heads = ioctl_geo.heads;
1675	geo->sectors = ioctl_geo.sectors;
1676	geo->cylinders = ioctl_geo.cylinders;
1677
1678	return 0;
1679	}
1680	#else /* __linux__ */
1681	static int hdev_probe_geometry(BlockDriverState bs, HDGeometry geo)
1682	{
1683	return -ENOTSUP;
1684	}
1685	#endif
1686
1687	#if defined(__linux__)
1688	static int handle_aiocb_ioctl(void *opaque)
1689	{
1690	RawPosixAIOData *aiocb = opaque;
1691	int ret;
1692
1693	ret = RETRY_ON_EINTR(
1694	ioctl(aiocb->aio_fildes, aiocb->ioctl.cmd, aiocb->ioctl.buf)
1695	);
1696	if (ret == -1) {
1697	return -errno;
1698	}
1699
1700	return 0;
1701	}
1702	#endif /* linux */
1703
1704	static int handle_aiocb_flush(void *opaque)
1705	{
1706	RawPosixAIOData *aiocb = opaque;
1707	BDRVRawState *s = aiocb->bs->opaque;
1708	int ret;
1709
1710	if (s->page_cache_inconsistent) {
1711	return -s->page_cache_inconsistent;
1712	}
1713
1714	ret = qemu_fdatasync(aiocb->aio_fildes);
1715	if (ret == -1) {
1716	trace_file_flush_fdatasync_failed(errno);
1717
1718	/* There is no clear definition of the semantics of a failing fsync(),
1719	* so we may have to assume the worst. The sad truth is that this
1720	* assumption is correct for Linux. Some pages are now probably marked
1721	* clean in the page cache even though they are inconsistent with the
1722	* on-disk contents. The next fdatasync() call would succeed, but no
1723	* further writeback attempt will be made. We can't get back to a state
1724	* in which we know what is on disk (we would have to rewrite
1725	* everything that was touched since the last fdatasync() at least), so
1726	* make bdrv_flush() fail permanently. Given that the behaviour isn't
1727	* really defined, I have little hope that other OSes are doing better.
1728	*
1729	* Obviously, this doesn't affect O_DIRECT, which bypasses the page
1730	* cache. */
1731	if ((s->open_flags & O_DIRECT) == 0) {
1732	s->page_cache_inconsistent = errno;
1733	}
1734	return -errno;
1735	}
1736	return 0;
1737	}
1738
1739	#ifdef CONFIG_PREADV
1740
1741	static bool preadv_present = true;
1742
1743	static ssize_t
1744	qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
1745	{
1746	return preadv(fd, iov, nr_iov, offset);
1747	}
1748
1749	static ssize_t
1750	qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
1751	{
1752	return pwritev(fd, iov, nr_iov, offset);
1753	}
1754
1755	#else
1756
1757	static bool preadv_present = false;
1758
1759	static ssize_t
1760	qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
1761	{
1762	return -ENOSYS;
1763	}
1764
1765	static ssize_t
1766	qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
1767	{
1768	return -ENOSYS;
1769	}
1770
1771	#endif
1772
1773	static ssize_t handle_aiocb_rw_vector(RawPosixAIOData *aiocb)
1774	{
1775	ssize_t len;
1776
1777	len = RETRY_ON_EINTR(
1778	(aiocb->aio_type & (QEMU_AIO_WRITE \| QEMU_AIO_ZONE_APPEND)) ?
1779	qemu_pwritev(aiocb->aio_fildes,
1780	aiocb->io.iov,
1781	aiocb->io.niov,
1782	aiocb->aio_offset) :
1783	qemu_preadv(aiocb->aio_fildes,
1784	aiocb->io.iov,
1785	aiocb->io.niov,
1786	aiocb->aio_offset)
1787	);
1788
1789	if (len == -1) {
1790	return -errno;
1791	}
1792	return len;
1793	}
1794
1795	/*
1796	* Read/writes the data to/from a given linear buffer.
1797	*
1798	* Returns the number of bytes handles or -errno in case of an error. Short
1799	* reads are only returned if the end of the file is reached.
1800	*/
1801	static ssize_t handle_aiocb_rw_linear(RawPosixAIOData aiocb, char buf)
1802	{
1803	ssize_t offset = 0;
1804	ssize_t len;
1805
1806	while (offset < aiocb->aio_nbytes) {
1807	if (aiocb->aio_type & (QEMU_AIO_WRITE \| QEMU_AIO_ZONE_APPEND)) {
1808	len = pwrite(aiocb->aio_fildes,
1809	(const char *)buf + offset,
1810	aiocb->aio_nbytes - offset,
1811	aiocb->aio_offset + offset);
1812	} else {
1813	len = pread(aiocb->aio_fildes,
1814	buf + offset,
1815	aiocb->aio_nbytes - offset,
1816	aiocb->aio_offset + offset);
1817	}
1818	if (len == -1 && errno == EINTR) {
1819	continue;
1820	} else if (len == -1 && errno == EINVAL &&
1821	(aiocb->bs->open_flags & BDRV_O_NOCACHE) &&
1822	!(aiocb->aio_type & QEMU_AIO_WRITE) &&
1823	offset > 0) {
1824	/* O_DIRECT pread() may fail with EINVAL when offset is unaligned
1825	* after a short read. Assume that O_DIRECT short reads only occur
1826	* at EOF. Therefore this is a short read, not an I/O error.
1827	*/
1828	break;
1829	} else if (len == -1) {
1830	offset = -errno;
1831	break;
1832	} else if (len == 0) {
1833	break;
1834	}
1835	offset += len;
1836	}
1837
1838	return offset;
1839	}
1840
1841	static int handle_aiocb_rw(void *opaque)
1842	{
1843	RawPosixAIOData *aiocb = opaque;
1844	ssize_t nbytes;
1845	char *buf;
1846
1847	if (!(aiocb->aio_type & QEMU_AIO_MISALIGNED)) {
1848	/*
1849	* If there is just a single buffer, and it is properly aligned
1850	* we can just use plain pread/pwrite without any problems.
1851	*/
1852	if (aiocb->io.niov == 1) {
1853	nbytes = handle_aiocb_rw_linear(aiocb, aiocb->io.iov->iov_base);
1854	goto out;
1855	}
1856	/*
1857	* We have more than one iovec, and all are properly aligned.
1858	*
1859	* Try preadv/pwritev first and fall back to linearizing the
1860	* buffer if it's not supported.
1861	*/
1862	if (preadv_present) {
1863	nbytes = handle_aiocb_rw_vector(aiocb);
1864	if (nbytes == aiocb->aio_nbytes \|\|
1865	(nbytes < 0 && nbytes != -ENOSYS)) {
1866	goto out;
1867	}
1868	preadv_present = false;
1869	}
1870
1871	/*
1872	* XXX(hch): short read/write. no easy way to handle the reminder
1873	* using these interfaces. For now retry using plain
1874	* pread/pwrite?
1875	*/
1876	}
1877
1878	/*
1879	* Ok, we have to do it the hard way, copy all segments into
1880	* a single aligned buffer.
1881	*/
1882	buf = qemu_try_blockalign(aiocb->bs, aiocb->aio_nbytes);
1883	if (buf == NULL) {
1884	nbytes = -ENOMEM;
1885	goto out;
1886	}
1887
1888	if (aiocb->aio_type & QEMU_AIO_WRITE) {
1889	char *p = buf;
1890	int i;
1891
1892	for (i = 0; i < aiocb->io.niov; ++i) {
1893	memcpy(p, aiocb->io.iov[i].iov_base, aiocb->io.iov[i].iov_len);
1894	p += aiocb->io.iov[i].iov_len;
1895	}
1896	assert(p - buf == aiocb->aio_nbytes);
1897	}
1898
1899	nbytes = handle_aiocb_rw_linear(aiocb, buf);
1900	if (!(aiocb->aio_type & (QEMU_AIO_WRITE \| QEMU_AIO_ZONE_APPEND))) {
1901	char *p = buf;
1902	size_t count = aiocb->aio_nbytes, copy;
1903	int i;
1904
1905	for (i = 0; i < aiocb->io.niov && count; ++i) {
1906	copy = count;
1907	if (copy > aiocb->io.iov[i].iov_len) {
1908	copy = aiocb->io.iov[i].iov_len;
1909	}
1910	memcpy(aiocb->io.iov[i].iov_base, p, copy);
1911	assert(count >= copy);
1912	p += copy;
1913	count -= copy;
1914	}
1915	assert(count == 0);
1916	}
1917	qemu_vfree(buf);
1918
1919	out:
1920	if (nbytes == aiocb->aio_nbytes) {
1921	return 0;
1922	} else if (nbytes >= 0 && nbytes < aiocb->aio_nbytes) {
1923	if (aiocb->aio_type & QEMU_AIO_WRITE) {
1924	return -EINVAL;
1925	} else {
1926	iov_memset(aiocb->io.iov, aiocb->io.niov, nbytes,
1927	0, aiocb->aio_nbytes - nbytes);
1928	return 0;
1929	}
1930	} else {
1931	assert(nbytes < 0);
1932	return nbytes;
1933	}
1934	}
1935
1936	#if defined(CONFIG_FALLOCATE) \|\| defined(BLKZEROOUT) \|\| defined(BLKDISCARD)
1937	static int translate_err(int err)
1938	{
1939	if (err == -ENODEV \|\| err == -ENOSYS \|\| err == -EOPNOTSUPP \|\|
1940	err == -ENOTTY) {
1941	err = -ENOTSUP;
1942	}
1943	return err;
1944	}
1945	#endif
1946
1947	#ifdef CONFIG_FALLOCATE
1948	static int do_fallocate(int fd, int mode, off_t offset, off_t len)
1949	{
1950	do {
1951	if (fallocate(fd, mode, offset, len) == 0) {
1952	return 0;
1953	}
1954	} while (errno == EINTR);
1955	return translate_err(-errno);
1956	}
1957	#endif
1958
1959	static ssize_t handle_aiocb_write_zeroes_block(RawPosixAIOData *aiocb)
1960	{
1961	int ret = -ENOTSUP;
1962	BDRVRawState *s = aiocb->bs->opaque;
1963
1964	if (!s->has_write_zeroes) {
1965	return -ENOTSUP;
1966	}
1967
1968	#ifdef BLKZEROOUT
1969	/* The BLKZEROOUT implementation in the kernel doesn't set
1970	* BLKDEV_ZERO_NOFALLBACK, so we can't call this if we have to avoid slow
1971	* fallbacks. */
1972	if (!(aiocb->aio_type & QEMU_AIO_NO_FALLBACK)) {
1973	do {
1974	uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
1975	if (ioctl(aiocb->aio_fildes, BLKZEROOUT, range) == 0) {
1976	return 0;
1977	}
1978	} while (errno == EINTR);
1979
1980	ret = translate_err(-errno);
1981	if (ret == -ENOTSUP) {
1982	s->has_write_zeroes = false;
1983	}
1984	}
1985	#endif
1986
1987	return ret;
1988	}
1989
1990	static int handle_aiocb_write_zeroes(void *opaque)
1991	{
1992	RawPosixAIOData *aiocb = opaque;
1993	#ifdef CONFIG_FALLOCATE
1994	BDRVRawState *s = aiocb->bs->opaque;
1995	int64_t len;
1996	#endif
1997
1998	if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
1999	return handle_aiocb_write_zeroes_block(aiocb);
2000	}
2001
2002	#ifdef CONFIG_FALLOCATE_ZERO_RANGE
2003	if (s->has_write_zeroes) {
2004	int ret = do_fallocate(s->fd, FALLOC_FL_ZERO_RANGE,
2005	aiocb->aio_offset, aiocb->aio_nbytes);
2006	if (ret == -ENOTSUP) {
2007	s->has_write_zeroes = false;
2008	} else if (ret == 0 \|\| ret != -EINVAL) {
2009	return ret;
2010	}
2011	/*
2012	* Note: Some file systems do not like unaligned byte ranges, and
2013	* return EINVAL in such a case, though they should not do it according
2014	* to the man-page of fallocate(). Thus we simply ignore this return
2015	* value and try the other fallbacks instead.
2016	*/
2017	}
2018	#endif
2019
2020	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
2021	if (s->has_discard && s->has_fallocate) {
2022	int ret = do_fallocate(s->fd,
2023	FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
2024	aiocb->aio_offset, aiocb->aio_nbytes);
2025	if (ret == 0) {
2026	ret = do_fallocate(s->fd, 0, aiocb->aio_offset, aiocb->aio_nbytes);
2027	if (ret == 0 \|\| ret != -ENOTSUP) {
2028	return ret;
2029	}
2030	s->has_fallocate = false;
2031	} else if (ret == -EINVAL) {
2032	/*
2033	* Some file systems like older versions of GPFS do not like un-
2034	* aligned byte ranges, and return EINVAL in such a case, though
2035	* they should not do it according to the man-page of fallocate().
2036	* Warn about the bad filesystem and try the final fallback instead.
2037	*/
2038	warn_report_once("Your file system is misbehaving: "
2039	"fallocate(FALLOC_FL_PUNCH_HOLE) returned EINVAL. "
2040	"Please report this bug to your file system "
2041	"vendor.");
2042	} else if (ret != -ENOTSUP) {
2043	return ret;
2044	} else {
2045	s->has_discard = false;
2046	}
2047	}
2048	#endif
2049
2050	#ifdef CONFIG_FALLOCATE
2051	/* Last resort: we are trying to extend the file with zeroed data. This
2052	* can be done via fallocate(fd, 0) */
2053	len = raw_getlength(aiocb->bs);
2054	if (s->has_fallocate && len >= 0 && aiocb->aio_offset >= len) {
2055	int ret = do_fallocate(s->fd, 0, aiocb->aio_offset, aiocb->aio_nbytes);
2056	if (ret == 0 \|\| ret != -ENOTSUP) {
2057	return ret;
2058	}
2059	s->has_fallocate = false;
2060	}
2061	#endif
2062
2063	return -ENOTSUP;
2064	}
2065
2066	static int handle_aiocb_write_zeroes_unmap(void *opaque)
2067	{
2068	RawPosixAIOData *aiocb = opaque;
2069	BDRVRawState *s G_GNUC_UNUSED = aiocb->bs->opaque;
2070
2071	/* First try to write zeros and unmap at the same time */
2072
2073	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
2074	int ret = do_fallocate(s->fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
2075	aiocb->aio_offset, aiocb->aio_nbytes);
2076	switch (ret) {
2077	case -ENOTSUP:
2078	case -EINVAL:
2079	case -EBUSY:
2080	break;
2081	default:
2082	return ret;
2083	}
2084	#endif
2085
2086	/* If we couldn't manage to unmap while guaranteed that the area reads as
2087	* all-zero afterwards, just write zeroes without unmapping */
2088	return handle_aiocb_write_zeroes(aiocb);
2089	}
2090
2091	#ifndef HAVE_COPY_FILE_RANGE
2092	#if !defined(EMSCRIPTEN) && !defined(__GNU__)
2093	static
2094	#endif
2095	ssize_t copy_file_range(int in_fd, off_t *in_off, int out_fd,
2096	off_t *out_off, size_t len, unsigned int flags)
2097	{
2098	#ifdef __NR_copy_file_range
2099	return syscall(__NR_copy_file_range, in_fd, in_off, out_fd,
2100	out_off, len, flags);
2101	#else
2102	errno = ENOSYS;
2103	return -1;
2104	#endif
2105	}
2106	#endif
2107
2108	/*
2109	* parse_zone - Fill a zone descriptor
2110	*/
2111	#if defined(CONFIG_BLKZONED)
2112	static inline int parse_zone(struct BlockZoneDescriptor *zone,
2113	const struct blk_zone *blkz) {
2114	zone->start = blkz->start << BDRV_SECTOR_BITS;
2115	zone->length = blkz->len << BDRV_SECTOR_BITS;
2116	zone->wp = blkz->wp << BDRV_SECTOR_BITS;
2117
2118	#ifdef HAVE_BLK_ZONE_REP_CAPACITY
2119	zone->cap = blkz->capacity << BDRV_SECTOR_BITS;
2120	#else
2121	zone->cap = blkz->len << BDRV_SECTOR_BITS;
2122	#endif
2123
2124	switch (blkz->type) {
2125	case BLK_ZONE_TYPE_SEQWRITE_REQ:
2126	zone->type = BLK_ZT_SWR;
2127	break;
2128	case BLK_ZONE_TYPE_SEQWRITE_PREF:
2129	zone->type = BLK_ZT_SWP;
2130	break;
2131	case BLK_ZONE_TYPE_CONVENTIONAL:
2132	zone->type = BLK_ZT_CONV;
2133	break;
2134	default:
2135	error_report("Unsupported zone type: 0x%x", blkz->type);
2136	return -ENOTSUP;
2137	}
2138
2139	switch (blkz->cond) {
2140	case BLK_ZONE_COND_NOT_WP:
2141	zone->state = BLK_ZS_NOT_WP;
2142	break;
2143	case BLK_ZONE_COND_EMPTY:
2144	zone->state = BLK_ZS_EMPTY;
2145	break;
2146	case BLK_ZONE_COND_IMP_OPEN:
2147	zone->state = BLK_ZS_IOPEN;
2148	break;
2149	case BLK_ZONE_COND_EXP_OPEN:
2150	zone->state = BLK_ZS_EOPEN;
2151	break;
2152	case BLK_ZONE_COND_CLOSED:
2153	zone->state = BLK_ZS_CLOSED;
2154	break;
2155	case BLK_ZONE_COND_READONLY:
2156	zone->state = BLK_ZS_RDONLY;
2157	break;
2158	case BLK_ZONE_COND_FULL:
2159	zone->state = BLK_ZS_FULL;
2160	break;
2161	case BLK_ZONE_COND_OFFLINE:
2162	zone->state = BLK_ZS_OFFLINE;
2163	break;
2164	default:
2165	error_report("Unsupported zone state: 0x%x", blkz->cond);
2166	return -ENOTSUP;
2167	}
2168	return 0;
2169	}
2170	#endif
2171
2172	#if defined(CONFIG_BLKZONED)
2173	static int handle_aiocb_zone_report(void *opaque)
2174	{
2175	RawPosixAIOData *aiocb = opaque;
2176	int fd = aiocb->aio_fildes;
2177	unsigned int *nr_zones = aiocb->zone_report.nr_zones;
2178	BlockZoneDescriptor *zones = aiocb->zone_report.zones;
2179	/* zoned block devices use 512-byte sectors */
2180	uint64_t sector = aiocb->aio_offset / 512;
2181
2182	struct blk_zone *blkz;
2183	size_t rep_size;
2184	unsigned int nrz;
2185	int ret;
2186	unsigned int n = 0, i = 0;
2187
2188	nrz = *nr_zones;
2189	rep_size = sizeof(struct blk_zone_report) + nrz * sizeof(struct blk_zone);
2190	g_autofree struct blk_zone_report *rep = NULL;
2191	rep = g_malloc(rep_size);
2192
2193	blkz = (struct blk_zone *)(rep + 1);
2194	while (n < nrz) {
2195	memset(rep, 0, rep_size);
2196	rep->sector = sector;
2197	rep->nr_zones = nrz - n;
2198
2199	do {
2200	ret = ioctl(fd, BLKREPORTZONE, rep);
2201	} while (ret != 0 && errno == EINTR);
2202	if (ret != 0) {
2203	error_report("%d: ioctl BLKREPORTZONE at %" PRId64 " failed %d",
2204	fd, sector, errno);
2205	return -errno;
2206	}
2207
2208	if (!rep->nr_zones) {
2209	break;
2210	}
2211
2212	for (i = 0; i < rep->nr_zones; i++, n++) {
2213	ret = parse_zone(&zones[n], &blkz[i]);
2214	if (ret != 0) {
2215	return ret;
2216	}
2217
2218	/* The next report should start after the last zone reported */
2219	sector = blkz[i].start + blkz[i].len;
2220	}
2221	}
2222
2223	*nr_zones = n;
2224	return 0;
2225	}
2226	#endif
2227
2228	#if defined(CONFIG_BLKZONED)
2229	static int handle_aiocb_zone_mgmt(void *opaque)
2230	{
2231	RawPosixAIOData *aiocb = opaque;
2232	int fd = aiocb->aio_fildes;
2233	uint64_t sector = aiocb->aio_offset / 512;
2234	int64_t nr_sectors = aiocb->aio_nbytes / 512;
2235	struct blk_zone_range range;
2236	int ret;
2237
2238	/* Execute the operation */
2239	range.sector = sector;
2240	range.nr_sectors = nr_sectors;
2241	do {
2242	ret = ioctl(fd, aiocb->zone_mgmt.op, &range);
2243	} while (ret != 0 && errno == EINTR);
2244
2245	return ret < 0 ? -errno : ret;
2246	}
2247	#endif
2248
2249	static int handle_aiocb_copy_range(void *opaque)
2250	{
2251	RawPosixAIOData *aiocb = opaque;
2252	uint64_t bytes = aiocb->aio_nbytes;
2253	off_t in_off = aiocb->aio_offset;
2254	off_t out_off = aiocb->copy_range.aio_offset2;
2255
2256	while (bytes) {
2257	ssize_t ret = copy_file_range(aiocb->aio_fildes, &in_off,
2258	aiocb->copy_range.aio_fd2, &out_off,
2259	bytes, 0);
2260	trace_file_copy_file_range(aiocb->bs, aiocb->aio_fildes, in_off,
2261	aiocb->copy_range.aio_fd2, out_off, bytes,
2262	0, ret);
2263	if (ret == 0) {
2264	/* No progress (e.g. when beyond EOF), let the caller fall back to
2265	* buffer I/O. */
2266	return -ENOSPC;
2267	}
2268	if (ret < 0) {
2269	switch (errno) {
2270	case ENOSYS:
2271	return -ENOTSUP;
2272	case EINTR:
2273	continue;
2274	default:
2275	return -errno;
2276	}
2277	}
2278	bytes -= ret;
2279	}
2280	return 0;
2281	}
2282
2283	static int handle_aiocb_discard(void *opaque)
2284	{
2285	RawPosixAIOData *aiocb = opaque;
2286	int ret = -ENOTSUP;
2287	BDRVRawState *s = aiocb->bs->opaque;
2288
2289	if (!s->has_discard) {
2290	return -ENOTSUP;
2291	}
2292
2293	if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
2294	#ifdef BLKDISCARD
2295	do {
2296	uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
2297	if (ioctl(aiocb->aio_fildes, BLKDISCARD, range) == 0) {
2298	return 0;
2299	}
2300	} while (errno == EINTR);
2301
2302	ret = translate_err(-errno);
2303	#endif
2304	} else {
2305	#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
2306	ret = do_fallocate(s->fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
2307	aiocb->aio_offset, aiocb->aio_nbytes);
2308	ret = translate_err(ret);
2309	#elif defined(__APPLE__) && (__MACH__)
2310	fpunchhole_t fpunchhole;
2311	fpunchhole.fp_flags = 0;
2312	fpunchhole.reserved = 0;
2313	fpunchhole.fp_offset = aiocb->aio_offset;
2314	fpunchhole.fp_length = aiocb->aio_nbytes;
2315	if (fcntl(s->fd, F_PUNCHHOLE, &fpunchhole) == -1) {
2316	ret = errno == ENODEV ? -ENOTSUP : -errno;
2317	} else {
2318	ret = 0;
2319	}
2320	#endif
2321	}
2322
2323	if (ret == -ENOTSUP) {
2324	s->has_discard = false;
2325	}
2326	return ret;
2327	}
2328
2329	/*
2330	* Help alignment probing by allocating the first block.
2331	*
2332	* When reading with direct I/O from unallocated area on Gluster backed by XFS,
2333	* reading succeeds regardless of request length. In this case we fallback to
2334	* safe alignment which is not optimal. Allocating the first block avoids this
2335	* fallback.
2336	*
2337	* fd may be opened with O_DIRECT, but we don't know the buffer alignment or
2338	* request alignment, so we use safe values.
2339	*
2340	* Returns: 0 on success, -errno on failure. Since this is an optimization,
2341	* caller may ignore failures.
2342	*/
2343	static int allocate_first_block(int fd, size_t max_size)
2344	{
2345	size_t write_size = (max_size < MAX_BLOCKSIZE)
2346	? BDRV_SECTOR_SIZE
2347	: MAX_BLOCKSIZE;
2348	size_t max_align = MAX(MAX_BLOCKSIZE, qemu_real_host_page_size());
2349	void *buf;
2350	ssize_t n;
2351	int ret;
2352
2353	buf = qemu_memalign(max_align, write_size);
2354	memset(buf, 0, write_size);
2355
2356	n = RETRY_ON_EINTR(pwrite(fd, buf, write_size, 0));
2357
2358	ret = (n == -1) ? -errno : 0;
2359
2360	qemu_vfree(buf);
2361	return ret;
2362	}
2363
2364	static int handle_aiocb_truncate(void *opaque)
2365	{
2366	RawPosixAIOData *aiocb = opaque;
2367	int result = 0;
2368	int64_t current_length = 0;
2369	char *buf = NULL;
2370	struct stat st;
2371	int fd = aiocb->aio_fildes;
2372	int64_t offset = aiocb->aio_offset;
2373	PreallocMode prealloc = aiocb->truncate.prealloc;
2374	Error **errp = aiocb->truncate.errp;
2375
2376	if (fstat(fd, &st) < 0) {
2377	result = -errno;
2378	error_setg_errno(errp, -result, "Could not stat file");
2379	return result;
2380	}
2381
2382	current_length = st.st_size;
2383	if (current_length > offset && prealloc != PREALLOC_MODE_OFF) {
2384	error_setg(errp, "Cannot use preallocation for shrinking files");
2385	return -ENOTSUP;
2386	}
2387
2388	switch (prealloc) {
2389	#ifdef CONFIG_POSIX_FALLOCATE
2390	case PREALLOC_MODE_FALLOC:
2391	/*
2392	* Truncating before posix_fallocate() makes it about twice slower on
2393	* file systems that do not support fallocate(), trying to check if a
2394	* block is allocated before allocating it, so don't do that here.
2395	*/
2396	if (offset != current_length) {
2397	result = -posix_fallocate(fd, current_length,
2398	offset - current_length);
2399	if (result != 0) {
2400	/* posix_fallocate() doesn't set errno. */
2401	error_setg_errno(errp, -result,
2402	"Could not preallocate new data");
2403	} else if (current_length == 0) {
2404	/*
2405	* posix_fallocate() uses fallocate() if the filesystem
2406	* supports it, or fallback to manually writing zeroes. If
2407	* fallocate() was used, unaligned reads from the fallocated
2408	* area in raw_probe_alignment() will succeed, hence we need to
2409	* allocate the first block.
2410	*
2411	* Optimize future alignment probing; ignore failures.
2412	*/
2413	allocate_first_block(fd, offset);
2414	}
2415	} else {
2416	result = 0;
2417	}
2418	goto out;
2419	#endif
2420	case PREALLOC_MODE_FULL:
2421	{
2422	int64_t num = 0, left = offset - current_length;
2423	off_t seek_result;
2424
2425	/*
2426	* Knowing the final size from the beginning could allow the file
2427	* system driver to do less allocations and possibly avoid
2428	* fragmentation of the file.
2429	*/
2430	if (ftruncate(fd, offset) != 0) {
2431	result = -errno;
2432	error_setg_errno(errp, -result, "Could not resize file");
2433	goto out;
2434	}
2435
2436	buf = g_malloc0(65536);
2437
2438	seek_result = lseek(fd, current_length, SEEK_SET);
2439	if (seek_result < 0) {
2440	result = -errno;
2441	error_setg_errno(errp, -result,
2442	"Failed to seek to the old end of file");
2443	goto out;
2444	}
2445
2446	while (left > 0) {
2447	num = MIN(left, 65536);
2448	result = write(fd, buf, num);
2449	if (result < 0) {
2450	if (errno == EINTR) {
2451	continue;
2452	}
2453	result = -errno;
2454	error_setg_errno(errp, -result,
2455	"Could not write zeros for preallocation");
2456	goto out;
2457	}
2458	left -= result;
2459	}
2460	if (result >= 0) {
2461	result = fsync(fd);
2462	if (result < 0) {
2463	result = -errno;
2464	error_setg_errno(errp, -result,
2465	"Could not flush file to disk");
2466	goto out;
2467	}
2468	}
2469	goto out;
2470	}
2471	case PREALLOC_MODE_OFF:
2472	if (ftruncate(fd, offset) != 0) {
2473	result = -errno;
2474	error_setg_errno(errp, -result, "Could not resize file");
2475	} else if (current_length == 0 && offset > current_length) {
2476	/* Optimize future alignment probing; ignore failures. */
2477	allocate_first_block(fd, offset);
2478	}
2479	return result;
2480	default:
2481	result = -ENOTSUP;
2482	error_setg(errp, "Unsupported preallocation mode: %s",
2483	PreallocMode_str(prealloc));
2484	return result;
2485	}
2486
2487	out:
2488	if (result < 0) {
2489	if (ftruncate(fd, current_length) < 0) {
2490	error_report("Failed to restore old file length: %s",
2491	strerror(errno));
2492	}
2493	}
2494
2495	g_free(buf);
2496	return result;
2497	}
2498
2499	static int coroutine_fn raw_thread_pool_submit(ThreadPoolFunc func, void *arg)
2500	{
2501	return thread_pool_submit_co(func, arg);
2502	}
2503
2504	/*
2505	* Check if all memory in this vector is sector aligned.
2506	*/
2507	static bool bdrv_qiov_is_aligned(BlockDriverState bs, QEMUIOVector qiov)
2508	{
2509	int i;
2510	size_t alignment = bdrv_min_mem_align(bs);
2511	size_t len = bs->bl.request_alignment;
2512	IO_CODE();
2513
2514	for (i = 0; i < qiov->niov; i++) {
2515	if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2516	return false;
2517	}
2518	if (qiov->iov[i].iov_len % len) {
2519	return false;
2520	}
2521	}
2522
2523	return true;
2524	}
2525
2526	#ifdef CONFIG_LINUX_AIO
2527	static inline bool raw_check_linux_aio(BDRVRawState *s)
2528	{
2529	Error *local_err = NULL;
2530	AioContext *ctx;
2531
2532	if (!s->use_linux_aio) {
2533	return false;
2534	}
2535
2536	ctx = qemu_get_current_aio_context();
2537	if (unlikely(!aio_setup_linux_aio(ctx, &local_err))) {
2538	error_reportf_err(local_err, "Unable to use Linux AIO, "
2539	"falling back to thread pool: ");
2540	s->use_linux_aio = false;
2541	return false;
2542	}
2543	return true;
2544	}
2545	#endif
2546
2547	static int coroutine_fn GRAPH_RDLOCK
2548	raw_co_prw(BlockDriverState bs, int64_t offset_ptr, uint64_t bytes,
2549	QEMUIOVector *qiov, int type, int flags)
2550	{
2551	BDRVRawState *s = bs->opaque;
2552	RawPosixAIOData acb;
2553	int ret;
2554	uint64_t offset = *offset_ptr;
2555
2556	if (fd_open(bs) < 0)
2557	return -EIO;
2558	#if defined(CONFIG_BLKZONED)
2559	if ((type & (QEMU_AIO_WRITE \| QEMU_AIO_ZONE_APPEND)) &&
2560	bs->bl.zoned != BLK_Z_NONE) {
2561	qemu_co_mutex_lock(&bs->wps->colock);
2562	if (type & QEMU_AIO_ZONE_APPEND) {
2563	int index = offset / bs->bl.zone_size;
2564	offset = bs->wps->wp[index];
2565	}
2566	}
2567	#endif
2568
2569	/*
2570	* When using O_DIRECT, the request must be aligned to be able to use
2571	* either libaio or io_uring interface. If not fail back to regular thread
2572	* pool read/write code which emulates this for us if we
2573	* set QEMU_AIO_MISALIGNED.
2574	*/
2575	if (s->needs_alignment && !bdrv_qiov_is_aligned(bs, qiov)) {
2576	type \|= QEMU_AIO_MISALIGNED;
2577	#ifdef CONFIG_LINUX_IO_URING
2578	} else if (s->use_linux_io_uring) {
2579	assert(qiov->size == bytes);
2580	ret = luring_co_submit(bs, s->fd, offset, qiov, type, flags);
2581	goto out;
2582	#endif
2583	#ifdef CONFIG_LINUX_AIO
2584	} else if (raw_check_linux_aio(s)) {
2585	assert(qiov->size == bytes);
2586	ret = laio_co_submit(s->fd, offset, qiov, type, flags,
2587	s->aio_max_batch);
2588	goto out;
2589	#endif
2590	}
2591
2592	acb = (RawPosixAIOData) {
2593	.bs = bs,
2594	.aio_fildes = s->fd,
2595	.aio_type = type,
2596	.aio_offset = offset,
2597	.aio_nbytes = bytes,
2598	.io = {
2599	.iov = qiov->iov,
2600	.niov = qiov->niov,
2601	},
2602	};
2603
2604	assert(qiov->size == bytes);
2605	ret = raw_thread_pool_submit(handle_aiocb_rw, &acb);
2606	if (ret == 0 && (flags & BDRV_REQ_FUA)) {
2607	/* TODO Use pwritev2() instead if it's available */
2608	ret = bdrv_co_flush(bs);
2609	}
2610	goto out; /* Avoid the compiler err of unused label */
2611
2612	out:
2613	#if defined(CONFIG_BLKZONED)
2614	if ((type & (QEMU_AIO_WRITE \| QEMU_AIO_ZONE_APPEND)) &&
2615	bs->bl.zoned != BLK_Z_NONE) {
2616	BlockZoneWps *wps = bs->wps;
2617	if (ret == 0) {
2618	uint64_t *wp = &wps->wp[offset / bs->bl.zone_size];
2619	if (!BDRV_ZT_IS_CONV(*wp)) {
2620	if (type & QEMU_AIO_ZONE_APPEND) {
2621	offset_ptr = wp;
2622	trace_zbd_zone_append_complete(bs, *offset_ptr
2623	>> BDRV_SECTOR_BITS);
2624	}
2625	/* Advance the wp if needed */
2626	if (offset + bytes > *wp) {
2627	*wp = offset + bytes;
2628	}
2629	}
2630	} else {
2631	/*
2632	* write and append write are not allowed to cross zone boundaries
2633	*/
2634	update_zones_wp(bs, s->fd, offset, 1);
2635	}
2636
2637	qemu_co_mutex_unlock(&wps->colock);
2638	}
2639	#endif
2640	return ret;
2641	}
2642
2643	static int coroutine_fn GRAPH_RDLOCK
2644	raw_co_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
2645	QEMUIOVector *qiov, BdrvRequestFlags flags)
2646	{
2647	return raw_co_prw(bs, &offset, bytes, qiov, QEMU_AIO_READ, flags);
2648	}
2649
2650	static int coroutine_fn GRAPH_RDLOCK
2651	raw_co_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes,
2652	QEMUIOVector *qiov, BdrvRequestFlags flags)
2653	{
2654	return raw_co_prw(bs, &offset, bytes, qiov, QEMU_AIO_WRITE, flags);
2655	}
2656
2657	static int coroutine_fn raw_co_flush_to_disk(BlockDriverState *bs)
2658	{
2659	BDRVRawState *s = bs->opaque;
2660	RawPosixAIOData acb;
2661	int ret;
2662
2663	ret = fd_open(bs);
2664	if (ret < 0) {
2665	return ret;
2666	}
2667
2668	acb = (RawPosixAIOData) {
2669	.bs = bs,
2670	.aio_fildes = s->fd,
2671	.aio_type = QEMU_AIO_FLUSH,
2672	};
2673
2674	#ifdef CONFIG_LINUX_IO_URING
2675	if (s->use_linux_io_uring) {
2676	return luring_co_submit(bs, s->fd, 0, NULL, QEMU_AIO_FLUSH, 0);
2677	}
2678	#endif
2679	#ifdef CONFIG_LINUX_AIO
2680	if (s->has_laio_fdsync && raw_check_linux_aio(s)) {
2681	return laio_co_submit(s->fd, 0, NULL, QEMU_AIO_FLUSH, 0, 0);
2682	}
2683	#endif
2684	return raw_thread_pool_submit(handle_aiocb_flush, &acb);
2685	}
2686
2687	static void raw_close(BlockDriverState *bs)
2688	{
2689	BDRVRawState *s = bs->opaque;
2690
2691	if (s->fd >= 0) {
2692	#if defined(CONFIG_BLKZONED)
2693	g_free(bs->wps);
2694	#endif
2695	qemu_close(s->fd);
2696	s->fd = -1;
2697	}
2698	}
2699
2700	/**
2701	* Truncates the given regular file @fd to @offset and, when growing, fills the
2702	* new space according to @prealloc.
2703	*
2704	* Returns: 0 on success, -errno on failure.
2705	*/
2706	static int coroutine_fn
2707	raw_regular_truncate(BlockDriverState *bs, int fd, int64_t offset,
2708	PreallocMode prealloc, Error **errp)
2709	{
2710	RawPosixAIOData acb;
2711
2712	acb = (RawPosixAIOData) {
2713	.bs = bs,
2714	.aio_fildes = fd,
2715	.aio_type = QEMU_AIO_TRUNCATE,
2716	.aio_offset = offset,
2717	.truncate = {
2718	.prealloc = prealloc,
2719	.errp = errp,
2720	},
2721	};
2722
2723	return raw_thread_pool_submit(handle_aiocb_truncate, &acb);
2724	}
2725
2726	static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
2727	bool exact, PreallocMode prealloc,
2728	BdrvRequestFlags flags, Error **errp)
2729	{
2730	BDRVRawState *s = bs->opaque;
2731	struct stat st;
2732	int ret;
2733
2734	if (fstat(s->fd, &st)) {
2735	ret = -errno;
2736	error_setg_errno(errp, -ret, "Failed to fstat() the file");
2737	return ret;
2738	}
2739
2740	if (S_ISREG(st.st_mode)) {
2741	/* Always resizes to the exact @offset */
2742	return raw_regular_truncate(bs, s->fd, offset, prealloc, errp);
2743	}
2744
2745	if (prealloc != PREALLOC_MODE_OFF) {
2746	error_setg(errp, "Preallocation mode '%s' unsupported for this "
2747	"non-regular file", PreallocMode_str(prealloc));
2748	return -ENOTSUP;
2749	}
2750
2751	if (S_ISCHR(st.st_mode) \|\| S_ISBLK(st.st_mode)) {
2752	int64_t cur_length = raw_getlength(bs);
2753
2754	if (offset != cur_length && exact) {
2755	error_setg(errp, "Cannot resize device files");
2756	return -ENOTSUP;
2757	} else if (offset > cur_length) {
2758	error_setg(errp, "Cannot grow device files");
2759	return -EINVAL;
2760	}
2761	} else {
2762	error_setg(errp, "Resizing this file is not supported");
2763	return -ENOTSUP;
2764	}
2765
2766	return 0;
2767	}
2768
2769	#ifdef __OpenBSD__
2770	static int64_t raw_getlength(BlockDriverState *bs)
2771	{
2772	BDRVRawState *s = bs->opaque;
2773	int fd = s->fd;
2774	struct stat st;
2775
2776	if (fstat(fd, &st))
2777	return -errno;
2778	if (S_ISCHR(st.st_mode) \|\| S_ISBLK(st.st_mode)) {
2779	struct disklabel dl;
2780
2781	if (ioctl(fd, DIOCGDINFO, &dl))
2782	return -errno;
2783	return (uint64_t)dl.d_secsize *
2784	dl.d_partitions[DISKPART(st.st_rdev)].p_size;
2785	} else
2786	return st.st_size;
2787	}
2788	#elif defined(__NetBSD__)
2789	static int64_t raw_getlength(BlockDriverState *bs)
2790	{
2791	BDRVRawState *s = bs->opaque;
2792	int fd = s->fd;
2793	struct stat st;
2794
2795	if (fstat(fd, &st))
2796	return -errno;
2797	if (S_ISCHR(st.st_mode) \|\| S_ISBLK(st.st_mode)) {
2798	struct dkwedge_info dkw;
2799
2800	if (ioctl(fd, DIOCGWEDGEINFO, &dkw) != -1) {
2801	return dkw.dkw_size * 512;
2802	} else {
2803	struct disklabel dl;
2804
2805	if (ioctl(fd, DIOCGDINFO, &dl))
2806	return -errno;
2807	return (uint64_t)dl.d_secsize *
2808	dl.d_partitions[DISKPART(st.st_rdev)].p_size;
2809	}
2810	} else
2811	return st.st_size;
2812	}
2813	#elif defined(__sun__)
2814	static int64_t raw_getlength(BlockDriverState *bs)
2815	{
2816	BDRVRawState *s = bs->opaque;
2817	struct dk_minfo minfo;
2818	int ret;
2819	int64_t size;
2820
2821	ret = fd_open(bs);
2822	if (ret < 0) {
2823	return ret;
2824	}
2825
2826	/*
2827	* Use the DKIOCGMEDIAINFO ioctl to read the size.
2828	*/
2829	ret = ioctl(s->fd, DKIOCGMEDIAINFO, &minfo);
2830	if (ret != -1) {
2831	return minfo.dki_lbsize * minfo.dki_capacity;
2832	}
2833
2834	/*
2835	* There are reports that lseek on some devices fails, but
2836	* irc discussion said that contingency on contingency was overkill.
2837	*/
2838	size = lseek(s->fd, 0, SEEK_END);
2839	if (size < 0) {
2840	return -errno;
2841	}
2842	return size;
2843	}
2844	#elif defined(CONFIG_BSD)
2845	static int64_t raw_getlength(BlockDriverState *bs)
2846	{
2847	BDRVRawState *s = bs->opaque;
2848	int fd = s->fd;
2849	int64_t size;
2850	struct stat sb;
2851	#if defined (__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
2852	int reopened = 0;
2853	#endif
2854	int ret;
2855
2856	ret = fd_open(bs);
2857	if (ret < 0)
2858	return ret;
2859
2860	#if defined (__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
2861	again:
2862	#endif
2863	if (!fstat(fd, &sb) && (S_IFCHR & sb.st_mode)) {
2864	size = 0;
2865	#ifdef DIOCGMEDIASIZE
2866	if (ioctl(fd, DIOCGMEDIASIZE, (off_t *)&size)) {
2867	size = 0;
2868	}
2869	#endif
2870	#ifdef DIOCGPART
2871	if (size == 0) {
2872	struct partinfo pi;
2873	if (ioctl(fd, DIOCGPART, &pi) == 0) {
2874	size = pi.media_size;
2875	}
2876	}
2877	#endif
2878	#if defined(DKIOCGETBLOCKCOUNT) && defined(DKIOCGETBLOCKSIZE)
2879	if (size == 0) {
2880	uint64_t sectors = 0;
2881	uint32_t sector_size = 0;
2882
2883	if (ioctl(fd, DKIOCGETBLOCKCOUNT, &sectors) == 0
2884	&& ioctl(fd, DKIOCGETBLOCKSIZE, &sector_size) == 0) {
2885	size = sectors * sector_size;
2886	}
2887	}
2888	#endif
2889	if (size == 0) {
2890	size = lseek(fd, 0LL, SEEK_END);
2891	}
2892	if (size < 0) {
2893	return -errno;
2894	}
2895	#if defined(__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
2896	switch(s->type) {
2897	case FTYPE_CD:
2898	/* XXX FreeBSD acd returns UINT_MAX sectors for an empty drive */
2899	if (size == 2048LL * (unsigned)-1)
2900	size = 0;
2901	/* XXX no disc? maybe we need to reopen... */
2902	if (size <= 0 && !reopened && cdrom_reopen(bs) >= 0) {
2903	reopened = 1;
2904	goto again;
2905	}
2906	}
2907	#endif
2908	} else {
2909	size = lseek(fd, 0, SEEK_END);
2910	if (size < 0) {
2911	return -errno;
2912	}
2913	}
2914	return size;
2915	}
2916	#else
2917	static int64_t raw_getlength(BlockDriverState *bs)
2918	{
2919	BDRVRawState *s = bs->opaque;
2920	int ret;
2921	int64_t size;
2922
2923	ret = fd_open(bs);
2924	if (ret < 0) {
2925	return ret;
2926	}
2927
2928	size = lseek(s->fd, 0, SEEK_END);
2929	if (size < 0) {
2930	return -errno;
2931	}
2932	return size;
2933	}
2934	#endif
2935
2936	static int64_t coroutine_fn raw_co_getlength(BlockDriverState *bs)
2937	{
2938	return raw_getlength(bs);
2939	}
2940
2941	static int64_t coroutine_fn raw_co_get_allocated_file_size(BlockDriverState *bs)
2942	{
2943	struct stat st;
2944	BDRVRawState *s = bs->opaque;
2945
2946	if (fstat(s->fd, &st) < 0) {
2947	return -errno;
2948	}
2949	return (int64_t)st.st_blocks * 512;
2950	}
2951
2952	static int coroutine_fn
2953	raw_co_create(BlockdevCreateOptions options, Error *errp)
2954	{
2955	BlockdevCreateOptionsFile *file_opts;
2956	Error *local_err = NULL;
2957	int fd;
2958	uint64_t perm, shared;
2959	int result = 0;
2960
2961	/* Validate options and set default values */
2962	assert(options->driver == BLOCKDEV_DRIVER_FILE);
2963	file_opts = &options->u.file;
2964
2965	if (!file_opts->has_nocow) {
2966	file_opts->nocow = false;
2967	}
2968	if (!file_opts->has_preallocation) {
2969	file_opts->preallocation = PREALLOC_MODE_OFF;
2970	}
2971	if (!file_opts->has_extent_size_hint) {
2972	file_opts->extent_size_hint = 1 * MiB;
2973	}
2974	if (file_opts->extent_size_hint > UINT32_MAX) {
2975	result = -EINVAL;
2976	error_setg(errp, "Extent size hint is too large");
2977	goto out;
2978	}
2979
2980	/* Create file */
2981	fd = qemu_create(file_opts->filename, O_RDWR \| O_BINARY, 0644, errp);
2982	if (fd < 0) {
2983	result = -errno;
2984	goto out;
2985	}
2986
2987	/* Take permissions: We want to discard everything, so we need
2988	* BLK_PERM_WRITE; and truncation to the desired size requires
2989	* BLK_PERM_RESIZE.
2990	* On the other hand, we cannot share the RESIZE permission
2991	* because we promise that after this function, the file has the
2992	* size given in the options. If someone else were to resize it
2993	* concurrently, we could not guarantee that.
2994	* Note that after this function, we can no longer guarantee that
2995	* the file is not touched by a third party, so it may be resized
2996	* then. */
2997	perm = BLK_PERM_WRITE \| BLK_PERM_RESIZE;
2998	shared = BLK_PERM_ALL & ~BLK_PERM_RESIZE;
2999
3000	/* Step one: Take locks */
3001	result = raw_apply_lock_bytes(NULL, fd, perm, ~shared, false, errp);
3002	if (result < 0) {
3003	goto out_close;
3004	}
3005
3006	/* Step two: Check that nobody else has taken conflicting locks */
3007	result = raw_check_lock_bytes(fd, perm, shared, errp);
3008	if (result < 0) {
3009	error_append_hint(errp,
3010	"Is another process using the image [%s]?\n",
3011	file_opts->filename);
3012	goto out_unlock;
3013	}
3014
3015	/* Clear the file by truncating it to 0 */
3016	result = raw_regular_truncate(NULL, fd, 0, PREALLOC_MODE_OFF, errp);
3017	if (result < 0) {
3018	goto out_unlock;
3019	}
3020
3021	if (file_opts->nocow) {
3022	#ifdef __linux__
3023	/* Set NOCOW flag to solve performance issue on fs like btrfs.
3024	* This is an optimisation. The FS_IOC_SETFLAGS ioctl return value
3025	* will be ignored since any failure of this operation should not
3026	* block the left work.
3027	*/
3028	int attr;
3029	if (ioctl(fd, FS_IOC_GETFLAGS, &attr) == 0) {
3030	attr \|= FS_NOCOW_FL;
3031	ioctl(fd, FS_IOC_SETFLAGS, &attr);
3032	}
3033	#endif
3034	}
3035	#ifdef FS_IOC_FSSETXATTR
3036	/*
3037	* Try to set the extent size hint. Failure is not fatal, and a warning is
3038	* only printed if the option was explicitly specified.
3039	*/
3040	{
3041	struct fsxattr attr;
3042	result = ioctl(fd, FS_IOC_FSGETXATTR, &attr);
3043	if (result == 0) {
3044	attr.fsx_xflags \|= FS_XFLAG_EXTSIZE;
3045	attr.fsx_extsize = file_opts->extent_size_hint;
3046	result = ioctl(fd, FS_IOC_FSSETXATTR, &attr);
3047	}
3048	if (result < 0 && file_opts->has_extent_size_hint &&
3049	file_opts->extent_size_hint)
3050	{
3051	warn_report("Failed to set extent size hint: %s",
3052	strerror(errno));
3053	}
3054	}
3055	#endif
3056
3057	/* Resize and potentially preallocate the file to the desired
3058	* final size */
3059	result = raw_regular_truncate(NULL, fd, file_opts->size,
3060	file_opts->preallocation, errp);
3061	if (result < 0) {
3062	goto out_unlock;
3063	}
3064
3065	out_unlock:
3066	raw_apply_lock_bytes(NULL, fd, 0, 0, true, &local_err);
3067	if (local_err) {
3068	/* The above call should not fail, and if it does, that does
3069	* not mean the whole creation operation has failed. So
3070	* report it the user for their convenience, but do not report
3071	* it to the caller. */
3072	warn_report_err(local_err);
3073	}
3074
3075	out_close:
3076	if (qemu_close(fd) != 0 && result == 0) {
3077	result = -errno;
3078	error_setg_errno(errp, -result, "Could not close the new file");
3079	}
3080	out:
3081	return result;
3082	}
3083
3084	static int coroutine_fn GRAPH_RDLOCK
3085	raw_co_create_opts(BlockDriver drv, const char filename,
3086	QemuOpts opts, Error *errp)
3087	{
3088	BlockdevCreateOptions options;
3089	int64_t total_size = 0;
3090	int64_t extent_size_hint = 0;
3091	bool has_extent_size_hint = false;
3092	bool nocow = false;
3093	PreallocMode prealloc;
3094	char *buf = NULL;
3095	Error *local_err = NULL;
3096
3097	/* Skip file: protocol prefix */
3098	strstart(filename, "file:", &filename);
3099
3100	/* Read out options */
3101	total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
3102	BDRV_SECTOR_SIZE);
3103	if (qemu_opt_get(opts, BLOCK_OPT_EXTENT_SIZE_HINT)) {
3104	has_extent_size_hint = true;
3105	extent_size_hint =
3106	qemu_opt_get_size_del(opts, BLOCK_OPT_EXTENT_SIZE_HINT, -1);
3107	}
3108	nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
3109	buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
3110	prealloc = qapi_enum_parse(&PreallocMode_lookup, buf,
3111	PREALLOC_MODE_OFF, &local_err);
3112	g_free(buf);
3113	if (local_err) {
3114	error_propagate(errp, local_err);
3115	return -EINVAL;
3116	}
3117
3118	options = (BlockdevCreateOptions) {
3119	.driver = BLOCKDEV_DRIVER_FILE,
3120	.u.file = {
3121	.filename = (char *) filename,
3122	.size = total_size,
3123	.has_preallocation = true,
3124	.preallocation = prealloc,
3125	.has_nocow = true,
3126	.nocow = nocow,
3127	.has_extent_size_hint = has_extent_size_hint,
3128	.extent_size_hint = extent_size_hint,
3129	},
3130	};
3131	return raw_co_create(&options, errp);
3132	}
3133
3134	static int coroutine_fn raw_co_delete_file(BlockDriverState *bs,
3135	Error **errp)
3136	{
3137	struct stat st;
3138	int ret;
3139
3140	if (!(stat(bs->filename, &st) == 0) \|\| !S_ISREG(st.st_mode)) {
3141	error_setg_errno(errp, ENOENT, "%s is not a regular file",
3142	bs->filename);
3143	return -ENOENT;
3144	}
3145
3146	ret = unlink(bs->filename);
3147	if (ret < 0) {
3148	ret = -errno;
3149	error_setg_errno(errp, -ret, "Error when deleting file %s",
3150	bs->filename);
3151	}
3152
3153	return ret;
3154	}
3155
3156	/*
3157	* Find allocation range in @bs around offset @start.
3158	* May change underlying file descriptor's file offset.
3159	* If @start is not in a hole, store @start in @data, and the
3160	* beginning of the next hole in @hole, and return 0.
3161	* If @start is in a non-trailing hole, store @start in @hole and the
3162	* beginning of the next non-hole in @data, and return 0.
3163	* If @start is in a trailing hole or beyond EOF, return -ENXIO.
3164	* If we can't find out, return a negative errno other than -ENXIO.
3165	*/
3166	static int find_allocation(BlockDriverState *bs, off_t start,
3167	off_t data, off_t hole)
3168	{
3169	#if defined SEEK_HOLE && defined SEEK_DATA
3170	BDRVRawState *s = bs->opaque;
3171	off_t offs;
3172
3173	/*
3174	* SEEK_DATA cases:
3175	* D1. offs == start: start is in data
3176	* D2. offs > start: start is in a hole, next data at offs
3177	* D3. offs < 0, errno = ENXIO: either start is in a trailing hole
3178	* or start is beyond EOF
3179	* If the latter happens, the file has been truncated behind
3180	* our back since we opened it. All bets are off then.
3181	* Treating like a trailing hole is simplest.
3182	* D4. offs < 0, errno != ENXIO: we learned nothing
3183	*/
3184	offs = lseek(s->fd, start, SEEK_DATA);
3185	if (offs < 0) {
3186	return -errno; /* D3 or D4 */
3187	}
3188
3189	if (offs < start) {
3190	/* This is not a valid return by lseek(). We are safe to just return
3191	* -EIO in this case, and we'll treat it like D4. */
3192	return -EIO;
3193	}
3194
3195	if (offs > start) {
3196	/* D2: in hole, next data at offs */
3197	*hole = start;
3198	*data = offs;
3199	return 0;
3200	}
3201
3202	/* D1: in data, end not yet known */
3203
3204	/*
3205	* SEEK_HOLE cases:
3206	* H1. offs == start: start is in a hole
3207	* If this happens here, a hole has been dug behind our back
3208	* since the previous lseek().
3209	* H2. offs > start: either start is in data, next hole at offs,
3210	* or start is in trailing hole, EOF at offs
3211	* Linux treats trailing holes like any other hole: offs ==
3212	* start. Solaris seeks to EOF instead: offs > start (blech).
3213	* If that happens here, a hole has been dug behind our back
3214	* since the previous lseek().
3215	* H3. offs < 0, errno = ENXIO: start is beyond EOF
3216	* If this happens, the file has been truncated behind our
3217	* back since we opened it. Treat it like a trailing hole.
3218	* H4. offs < 0, errno != ENXIO: we learned nothing
3219	* Pretend we know nothing at all, i.e. "forget" about D1.
3220	*/
3221	offs = lseek(s->fd, start, SEEK_HOLE);
3222	if (offs < 0) {
3223	return -errno; /* D1 and (H3 or H4) */
3224	}
3225
3226	if (offs < start) {
3227	/* This is not a valid return by lseek(). We are safe to just return
3228	* -EIO in this case, and we'll treat it like H4. */
3229	return -EIO;
3230	}
3231
3232	if (offs > start) {
3233	/*
3234	* D1 and H2: either in data, next hole at offs, or it was in
3235	* data but is now in a trailing hole. In the latter case,
3236	* all bets are off. Treating it as if it there was data all
3237	* the way to EOF is safe, so simply do that.
3238	*/
3239	*data = start;
3240	*hole = offs;
3241	return 0;
3242	}
3243
3244	/* D1 and H1 */
3245	return -EBUSY;
3246	#else
3247	return -ENOTSUP;
3248	#endif
3249	}
3250
3251	/*
3252	* Returns the allocation status of the specified offset.
3253	*
3254	* The block layer guarantees 'offset' and 'bytes' are within bounds.
3255	*
3256	* 'pnum' is set to the number of bytes (including and immediately following
3257	* the specified offset) that are known to be in the same
3258	* allocated/unallocated state.
3259	*
3260	* 'bytes' is a soft cap for 'pnum'. If the information is free, 'pnum' may
3261	* well exceed it.
3262	*/
3263	static int coroutine_fn raw_co_block_status(BlockDriverState *bs,
3264	unsigned int mode,
3265	int64_t offset,
3266	int64_t bytes, int64_t *pnum,
3267	int64_t *map,
3268	BlockDriverState **file)
3269	{
3270	off_t data = 0, hole = 0;
3271	int ret;
3272
3273	assert(QEMU_IS_ALIGNED(offset \| bytes, bs->bl.request_alignment));
3274
3275	ret = fd_open(bs);
3276	if (ret < 0) {
3277	return ret;
3278	}
3279
3280	if (!(mode & BDRV_WANT_ZERO)) {
3281	/* There is no backing file - all bytes are allocated in this file. */
3282	*pnum = bytes;
3283	*map = offset;
3284	*file = bs;
3285	return BDRV_BLOCK_DATA \| BDRV_BLOCK_OFFSET_VALID;
3286	}
3287
3288	ret = find_allocation(bs, offset, &data, &hole);
3289	if (ret == -ENXIO) {
3290	/* Trailing hole */
3291	*pnum = bytes;
3292	ret = BDRV_BLOCK_ZERO;
3293	} else if (ret < 0) {
3294	/* No info available, so pretend there are no holes */
3295	*pnum = bytes;
3296	ret = BDRV_BLOCK_DATA;
3297	} else if (data == offset) {
3298	/* On a data extent, compute bytes to the end of the extent,
3299	* possibly including a partial sector at EOF. */
3300	*pnum = hole - offset;
3301
3302	/*
3303	* We are not allowed to return partial sectors, though, so
3304	* round up if necessary.
3305	*/
3306	if (!QEMU_IS_ALIGNED(*pnum, bs->bl.request_alignment)) {
3307	int64_t file_length = raw_getlength(bs);
3308	if (file_length > 0) {
3309	/* Ignore errors, this is just a safeguard */
3310	assert(hole == file_length);
3311	}
3312	pnum = ROUND_UP(pnum, bs->bl.request_alignment);
3313	}
3314
3315	ret = BDRV_BLOCK_DATA;
3316	} else {
3317	/* On a hole, compute bytes to the beginning of the next extent. */
3318	assert(hole == offset);
3319	*pnum = data - offset;
3320	ret = BDRV_BLOCK_ZERO;
3321	}
3322	*map = offset;
3323	*file = bs;
3324	return ret \| BDRV_BLOCK_OFFSET_VALID;
3325	}
3326
3327	#if defined(__linux__)
3328	/* Verify that the file is not in the page cache */
3329	static void check_cache_dropped(BlockDriverState bs, Error *errp)
3330	{
3331	const size_t window_size = 128 * 1024 * 1024;
3332	BDRVRawState *s = bs->opaque;
3333	void *window = NULL;
3334	size_t length = 0;
3335	unsigned char *vec;
3336	size_t page_size;
3337	off_t offset;
3338	off_t end;
3339
3340	/* mincore(2) page status information requires 1 byte per page */
3341	page_size = sysconf(_SC_PAGESIZE);
3342	vec = g_malloc(DIV_ROUND_UP(window_size, page_size));
3343
3344	end = raw_getlength(bs);
3345
3346	for (offset = 0; offset < end; offset += window_size) {
3347	void *new_window;
3348	size_t new_length;
3349	size_t vec_end;
3350	size_t i;
3351	int ret;
3352
3353	/* Unmap previous window if size has changed */
3354	new_length = MIN(end - offset, window_size);
3355	if (new_length != length) {
3356	munmap(window, length);
3357	window = NULL;
3358	length = 0;
3359	}
3360
3361	new_window = mmap(window, new_length, PROT_NONE, MAP_PRIVATE,
3362	s->fd, offset);
3363	if (new_window == MAP_FAILED) {
3364	error_setg_errno(errp, errno, "mmap failed");
3365	break;
3366	}
3367
3368	window = new_window;
3369	length = new_length;
3370
3371	ret = mincore(window, length, vec);
3372	if (ret < 0) {
3373	error_setg_errno(errp, errno, "mincore failed");
3374	break;
3375	}
3376
3377	vec_end = DIV_ROUND_UP(length, page_size);
3378	for (i = 0; i < vec_end; i++) {
3379	if (vec[i] & 0x1) {
3380	break;
3381	}
3382	}
3383	if (i < vec_end) {
3384	error_setg(errp, "page cache still in use!");
3385	break;
3386	}
3387	}
3388
3389	if (window) {
3390	munmap(window, length);
3391	}
3392
3393	g_free(vec);
3394	}
3395	#endif /* __linux__ */
3396
3397	static void coroutine_fn GRAPH_RDLOCK
3398	raw_co_invalidate_cache(BlockDriverState bs, Error *errp)
3399	{
3400	BDRVRawState *s = bs->opaque;
3401	int ret;
3402
3403	ret = fd_open(bs);
3404	if (ret < 0) {
3405	error_setg_errno(errp, -ret, "The file descriptor is not open");
3406	return;
3407	}
3408
3409	if (!s->drop_cache) {
3410	return;
3411	}
3412
3413	if (s->open_flags & O_DIRECT) {
3414	return; /* No host kernel page cache */
3415	}
3416
3417	#if defined(__linux__)
3418	/* This sets the scene for the next syscall... */
3419	ret = bdrv_co_flush(bs);
3420	if (ret < 0) {
3421	error_setg_errno(errp, -ret, "flush failed");
3422	return;
3423	}
3424
3425	/* Linux does not invalidate pages that are dirty, locked, or mmapped by a
3426	* process. These limitations are okay because we just fsynced the file,
3427	* we don't use mmap, and the file should not be in use by other processes.
3428	*/
3429	ret = posix_fadvise(s->fd, 0, 0, POSIX_FADV_DONTNEED);
3430	if (ret != 0) { /* the return value is a positive errno */
3431	error_setg_errno(errp, ret, "fadvise failed");
3432	return;
3433	}
3434
3435	if (s->check_cache_dropped) {
3436	check_cache_dropped(bs, errp);
3437	}
3438	#else /* __linux__ */
3439	/* Do nothing. Live migration to a remote host with cache.direct=off is
3440	* unsupported on other host operating systems. Cache consistency issues
3441	* may occur but no error is reported here, partly because that's the
3442	* historical behavior and partly because it's hard to differentiate valid
3443	* configurations that should not cause errors.
3444	*/
3445	#endif /* !__linux__ */
3446	}
3447
3448	static void raw_account_discard(BDRVRawState *s, uint64_t nbytes, int ret)
3449	{
3450	if (ret) {
3451	s->stats.discard_nb_failed++;
3452	} else {
3453	s->stats.discard_nb_ok++;
3454	s->stats.discard_bytes_ok += nbytes;
3455	}
3456	}
3457
3458	/*
3459	* zone report - Get a zone block device's information in the form
3460	* of an array of zone descriptors.
3461	* zones is an array of zone descriptors to hold zone information on reply;
3462	* offset can be any byte within the entire size of the device;
3463	* nr_zones is the maximum number of sectors the command should operate on.
3464	*/
3465	#if defined(CONFIG_BLKZONED)
3466	static int coroutine_fn raw_co_zone_report(BlockDriverState *bs, int64_t offset,
3467	unsigned int *nr_zones,
3468	BlockZoneDescriptor *zones) {
3469	BDRVRawState *s = bs->opaque;
3470	RawPosixAIOData acb = (RawPosixAIOData) {
3471	.bs = bs,
3472	.aio_fildes = s->fd,
3473	.aio_type = QEMU_AIO_ZONE_REPORT,
3474	.aio_offset = offset,
3475	.zone_report = {
3476	.nr_zones = nr_zones,
3477	.zones = zones,
3478	},
3479	};
3480
3481	trace_zbd_zone_report(bs, *nr_zones, offset >> BDRV_SECTOR_BITS);
3482	return raw_thread_pool_submit(handle_aiocb_zone_report, &acb);
3483	}
3484	#endif
3485
3486	/*
3487	* zone management operations - Execute an operation on a zone
3488	*/
3489	#if defined(CONFIG_BLKZONED)
3490	static int coroutine_fn raw_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op,
3491	int64_t offset, int64_t len) {
3492	BDRVRawState *s = bs->opaque;
3493	RawPosixAIOData acb;
3494	int64_t zone_size, zone_size_mask;
3495	const char *op_name;
3496	unsigned long zo;
3497	int ret;
3498	BlockZoneWps *wps = bs->wps;
3499	int64_t capacity = bs->total_sectors << BDRV_SECTOR_BITS;
3500
3501	zone_size = bs->bl.zone_size;
3502	zone_size_mask = zone_size - 1;
3503	if (offset & zone_size_mask) {
3504	error_report("sector offset %" PRId64 " is not aligned to zone size "
3505	"%" PRId64 "", offset / 512, zone_size / 512);
3506	return -EINVAL;
3507	}
3508
3509	if (((offset + len) < capacity && len & zone_size_mask) \|\|
3510	offset + len > capacity) {
3511	error_report("number of sectors %" PRId64 " is not aligned to zone size"
3512	" %" PRId64 "", len / 512, zone_size / 512);
3513	return -EINVAL;
3514	}
3515
3516	uint32_t i = offset / bs->bl.zone_size;
3517	uint32_t nrz = len / bs->bl.zone_size;
3518	uint64_t *wp = &wps->wp[i];
3519	if (BDRV_ZT_IS_CONV(*wp) && len != capacity) {
3520	error_report("zone mgmt operations are not allowed for conventional zones");
3521	return -EIO;
3522	}
3523
3524	switch (op) {
3525	case BLK_ZO_OPEN:
3526	op_name = "BLKOPENZONE";
3527	zo = BLKOPENZONE;
3528	break;
3529	case BLK_ZO_CLOSE:
3530	op_name = "BLKCLOSEZONE";
3531	zo = BLKCLOSEZONE;
3532	break;
3533	case BLK_ZO_FINISH:
3534	op_name = "BLKFINISHZONE";
3535	zo = BLKFINISHZONE;
3536	break;
3537	case BLK_ZO_RESET:
3538	op_name = "BLKRESETZONE";
3539	zo = BLKRESETZONE;
3540	break;
3541	default:
3542	error_report("Unsupported zone op: 0x%x", op);
3543	return -ENOTSUP;
3544	}
3545
3546	acb = (RawPosixAIOData) {
3547	.bs = bs,
3548	.aio_fildes = s->fd,
3549	.aio_type = QEMU_AIO_ZONE_MGMT,
3550	.aio_offset = offset,
3551	.aio_nbytes = len,
3552	.zone_mgmt = {
3553	.op = zo,
3554	},
3555	};
3556
3557	trace_zbd_zone_mgmt(bs, op_name, offset >> BDRV_SECTOR_BITS,
3558	len >> BDRV_SECTOR_BITS);
3559	ret = raw_thread_pool_submit(handle_aiocb_zone_mgmt, &acb);
3560	if (ret != 0) {
3561	update_zones_wp(bs, s->fd, offset, nrz);
3562	error_report("ioctl %s failed %d", op_name, ret);
3563	return ret;
3564	}
3565
3566	if (zo == BLKRESETZONE && len == capacity) {
3567	ret = get_zones_wp(bs, s->fd, 0, bs->bl.nr_zones, 1);
3568	if (ret < 0) {
3569	error_report("reporting single wp failed");
3570	return ret;
3571	}
3572	} else if (zo == BLKRESETZONE) {
3573	for (unsigned int j = 0; j < nrz; ++j) {
3574	wp[j] = offset + j * zone_size;
3575	}
3576	} else if (zo == BLKFINISHZONE) {
3577	for (unsigned int j = 0; j < nrz; ++j) {
3578	/* The zoned device allows the last zone smaller that the
3579	* zone size. */
3580	wp[j] = MIN(offset + (j + 1) * zone_size, offset + len);
3581	}
3582	}
3583
3584	return ret;
3585	}
3586	#endif
3587
3588	#if defined(CONFIG_BLKZONED)
3589	static int coroutine_fn GRAPH_RDLOCK
3590	raw_co_zone_append(BlockDriverState *bs,
3591	int64_t *offset,
3592	QEMUIOVector *qiov,
3593	BdrvRequestFlags flags) {
3594	assert(flags == 0);
3595	int64_t zone_size_mask = bs->bl.zone_size - 1;
3596	int64_t iov_len = 0;
3597	int64_t len = 0;
3598
3599	if (*offset & zone_size_mask) {
3600	error_report("sector offset %" PRId64 " is not aligned to zone size "
3601	"%" PRId32 "", *offset / 512, bs->bl.zone_size / 512);
3602	return -EINVAL;
3603	}
3604
3605	int64_t wg = bs->bl.write_granularity;
3606	int64_t wg_mask = wg - 1;
3607	for (int i = 0; i < qiov->niov; i++) {
3608	iov_len = qiov->iov[i].iov_len;
3609	if (iov_len & wg_mask) {
3610	error_report("len of IOVector[%d] %" PRId64 " is not aligned to "
3611	"block size %" PRId64 "", i, iov_len, wg);
3612	return -EINVAL;
3613	}
3614	len += iov_len;
3615	}
3616
3617	trace_zbd_zone_append(bs, *offset >> BDRV_SECTOR_BITS);
3618	return raw_co_prw(bs, offset, len, qiov, QEMU_AIO_ZONE_APPEND, 0);
3619	}
3620	#endif
3621
3622	static coroutine_fn int
3623	raw_do_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes,
3624	bool blkdev)
3625	{
3626	BDRVRawState *s = bs->opaque;
3627	RawPosixAIOData acb;
3628	int ret;
3629
3630	acb = (RawPosixAIOData) {
3631	.bs = bs,
3632	.aio_fildes = s->fd,
3633	.aio_type = QEMU_AIO_DISCARD,
3634	.aio_offset = offset,
3635	.aio_nbytes = bytes,
3636	};
3637
3638	if (blkdev) {
3639	acb.aio_type \|= QEMU_AIO_BLKDEV;
3640	}
3641
3642	ret = raw_thread_pool_submit(handle_aiocb_discard, &acb);
3643	raw_account_discard(s, bytes, ret);
3644	return ret;
3645	}
3646
3647	static coroutine_fn int
3648	raw_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
3649	{
3650	return raw_do_pdiscard(bs, offset, bytes, false);
3651	}
3652
3653	static int coroutine_fn
3654	raw_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes,
3655	BdrvRequestFlags flags, bool blkdev)
3656	{
3657	BDRVRawState *s = bs->opaque;
3658	RawPosixAIOData acb;
3659	ThreadPoolFunc *handler;
3660
3661	#ifdef CONFIG_FALLOCATE
3662	if (offset + bytes > bs->total_sectors * BDRV_SECTOR_SIZE) {
3663	BdrvTrackedRequest *req;
3664
3665	/*
3666	* This is a workaround for a bug in the Linux XFS driver,
3667	* where writes submitted through the AIO interface will be
3668	* discarded if they happen beyond a concurrently running
3669	* fallocate() that increases the file length (i.e., both the
3670	* write and the fallocate() happen beyond the EOF).
3671	*
3672	* To work around it, we extend the tracked request for this
3673	* zero write until INT64_MAX (effectively infinity), and mark
3674	* it as serializing.
3675	*
3676	* We have to enable this workaround for all filesystems and
3677	* AIO modes (not just XFS with aio=native), because for
3678	* remote filesystems we do not know the host configuration.
3679	*/
3680
3681	req = bdrv_co_get_self_request(bs);
3682	assert(req);
3683	assert(req->type == BDRV_TRACKED_WRITE);
3684	assert(req->offset <= offset);
3685	assert(req->offset + req->bytes >= offset + bytes);
3686
3687	req->bytes = BDRV_MAX_LENGTH - req->offset;
3688
3689	bdrv_check_request(req->offset, req->bytes, &error_abort);
3690
3691	bdrv_make_request_serialising(req, bs->bl.request_alignment);
3692	}
3693	#endif
3694
3695	acb = (RawPosixAIOData) {
3696	.bs = bs,
3697	.aio_fildes = s->fd,
3698	.aio_type = QEMU_AIO_WRITE_ZEROES,
3699	.aio_offset = offset,
3700	.aio_nbytes = bytes,
3701	};
3702
3703	if (blkdev) {
3704	acb.aio_type \|= QEMU_AIO_BLKDEV;
3705	}
3706	if (flags & BDRV_REQ_NO_FALLBACK) {
3707	acb.aio_type \|= QEMU_AIO_NO_FALLBACK;
3708	}
3709
3710	if (flags & BDRV_REQ_MAY_UNMAP) {
3711	acb.aio_type \|= QEMU_AIO_DISCARD;
3712	handler = handle_aiocb_write_zeroes_unmap;
3713	} else {
3714	handler = handle_aiocb_write_zeroes;
3715	}
3716
3717	return raw_thread_pool_submit(handler, &acb);
3718	}
3719
3720	static int coroutine_fn raw_co_pwrite_zeroes(
3721	BlockDriverState *bs, int64_t offset,
3722	int64_t bytes, BdrvRequestFlags flags)
3723	{
3724	return raw_do_pwrite_zeroes(bs, offset, bytes, flags, false);
3725	}
3726
3727	static int coroutine_fn
3728	raw_co_get_info(BlockDriverState bs, BlockDriverInfo bdi)
3729	{
3730	return 0;
3731	}
3732
3733	static ImageInfoSpecific raw_get_specific_info(BlockDriverState bs,
3734	Error **errp)
3735	{
3736	ImageInfoSpecificFile *file_info = g_new0(ImageInfoSpecificFile, 1);
3737	ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
3738
3739	*spec_info = (ImageInfoSpecific){
3740	.type = IMAGE_INFO_SPECIFIC_KIND_FILE,
3741	.u.file.data = file_info,
3742	};
3743
3744	#ifdef FS_IOC_FSGETXATTR
3745	{
3746	BDRVRawState *s = bs->opaque;
3747	struct fsxattr attr;
3748	int ret;
3749
3750	ret = ioctl(s->fd, FS_IOC_FSGETXATTR, &attr);
3751	if (!ret && attr.fsx_extsize != 0) {
3752	file_info->has_extent_size_hint = true;
3753	file_info->extent_size_hint = attr.fsx_extsize;
3754	}
3755	}
3756	#endif
3757
3758	return spec_info;
3759	}
3760
3761	static BlockStatsSpecificFile get_blockstats_specific_file(BlockDriverState *bs)
3762	{
3763	BDRVRawState *s = bs->opaque;
3764	return (BlockStatsSpecificFile) {
3765	.discard_nb_ok = s->stats.discard_nb_ok,
3766	.discard_nb_failed = s->stats.discard_nb_failed,
3767	.discard_bytes_ok = s->stats.discard_bytes_ok,
3768	};
3769	}
3770
3771	static BlockStatsSpecific raw_get_specific_stats(BlockDriverState bs)
3772	{
3773	BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);
3774
3775	stats->driver = BLOCKDEV_DRIVER_FILE;
3776	stats->u.file = get_blockstats_specific_file(bs);
3777
3778	return stats;
3779	}
3780
3781	#if defined(HAVE_HOST_BLOCK_DEVICE)
3782	static BlockStatsSpecific hdev_get_specific_stats(BlockDriverState bs)
3783	{
3784	BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);
3785
3786	stats->driver = BLOCKDEV_DRIVER_HOST_DEVICE;
3787	stats->u.host_device = get_blockstats_specific_file(bs);
3788
3789	return stats;
3790	}
3791	#endif /* HAVE_HOST_BLOCK_DEVICE */
3792
3793	static QemuOptsList raw_create_opts = {
3794	.name = "raw-create-opts",
3795	.head = QTAILQ_HEAD_INITIALIZER(raw_create_opts.head),
3796	.desc = {
3797	{
3798	.name = BLOCK_OPT_SIZE,
3799	.type = QEMU_OPT_SIZE,
3800	.help = "Virtual disk size"
3801	},
3802	{
3803	.name = BLOCK_OPT_NOCOW,
3804	.type = QEMU_OPT_BOOL,
3805	.help = "Turn off copy-on-write (valid only on btrfs)"
3806	},
3807	{
3808	.name = BLOCK_OPT_PREALLOC,
3809	.type = QEMU_OPT_STRING,
3810	.help = "Preallocation mode (allowed values: off"
3811	#ifdef CONFIG_POSIX_FALLOCATE
3812	", falloc"
3813	#endif
3814	", full)"
3815	},
3816	{
3817	.name = BLOCK_OPT_EXTENT_SIZE_HINT,
3818	.type = QEMU_OPT_SIZE,
3819	.help = "Extent size hint for the image file, 0 to disable"
3820	},
3821	{ /* end of list */ }
3822	}
3823	};
3824
3825	static int raw_check_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared,
3826	Error **errp)
3827	{
3828	BDRVRawState *s = bs->opaque;
3829	int input_flags = s->reopen_state ? s->reopen_state->flags : bs->open_flags;
3830	int open_flags;
3831	int ret;
3832
3833	/* We may need a new fd if auto-read-only switches the mode */
3834	ret = raw_reconfigure_getfd(bs, input_flags, &open_flags, perm, errp);
3835	if (ret < 0) {
3836	return ret;
3837	} else if (ret != s->fd) {
3838	Error *local_err = NULL;
3839
3840	/*
3841	* Fail already check_perm() if we can't get a working O_DIRECT
3842	* alignment with the new fd.
3843	*/
3844	raw_probe_alignment(bs, ret, &local_err);
3845	if (local_err) {
3846	error_propagate(errp, local_err);
3847	return -EINVAL;
3848	}
3849
3850	s->perm_change_fd = ret;
3851	s->perm_change_flags = open_flags;
3852	}
3853
3854	/* Prepare permissions on old fd to avoid conflicts between old and new,
3855	* but keep everything locked that new will need. */
3856	ret = raw_handle_perm_lock(bs, RAW_PL_PREPARE, perm, shared, errp);
3857	if (ret < 0) {
3858	goto fail;
3859	}
3860
3861	/* Copy locks to the new fd */
3862	if (s->perm_change_fd && s->use_lock) {
3863	ret = raw_apply_lock_bytes(NULL, s->perm_change_fd, perm, ~shared,
3864	false, errp);
3865	if (ret < 0) {
3866	raw_handle_perm_lock(bs, RAW_PL_ABORT, 0, 0, NULL);
3867	goto fail;
3868	}
3869	}
3870	return 0;
3871
3872	fail:
3873	if (s->perm_change_fd) {
3874	qemu_close(s->perm_change_fd);
3875	}
3876	s->perm_change_fd = 0;
3877	return ret;
3878	}
3879
3880	static void raw_set_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared)
3881	{
3882	BDRVRawState *s = bs->opaque;
3883
3884	/* For reopen, we have already switched to the new fd (.bdrv_set_perm is
3885	* called after .bdrv_reopen_commit) */
3886	if (s->perm_change_fd && s->fd != s->perm_change_fd) {
3887	qemu_close(s->fd);
3888	s->fd = s->perm_change_fd;
3889	s->open_flags = s->perm_change_flags;
3890	}
3891	s->perm_change_fd = 0;
3892
3893	raw_handle_perm_lock(bs, RAW_PL_COMMIT, perm, shared, NULL);
3894	s->perm = perm;
3895	s->shared_perm = shared;
3896	}
3897
3898	static void raw_abort_perm_update(BlockDriverState *bs)
3899	{
3900	BDRVRawState *s = bs->opaque;
3901
3902	/* For reopen, .bdrv_reopen_abort is called afterwards and will close
3903	* the file descriptor. */
3904	if (s->perm_change_fd) {
3905	qemu_close(s->perm_change_fd);
3906	}
3907	s->perm_change_fd = 0;
3908
3909	raw_handle_perm_lock(bs, RAW_PL_ABORT, 0, 0, NULL);
3910	}
3911
3912	static int coroutine_fn GRAPH_RDLOCK raw_co_copy_range_from(
3913	BlockDriverState bs, BdrvChild src, int64_t src_offset,
3914	BdrvChild *dst, int64_t dst_offset, int64_t bytes,
3915	BdrvRequestFlags read_flags, BdrvRequestFlags write_flags)
3916	{
3917	return bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3918	read_flags, write_flags);
3919	}
3920
3921	static int coroutine_fn GRAPH_RDLOCK
3922	raw_co_copy_range_to(BlockDriverState *bs,
3923	BdrvChild *src, int64_t src_offset,
3924	BdrvChild *dst, int64_t dst_offset,
3925	int64_t bytes, BdrvRequestFlags read_flags,
3926	BdrvRequestFlags write_flags)
3927	{
3928	RawPosixAIOData acb;
3929	BDRVRawState *s = bs->opaque;
3930	BDRVRawState *src_s;
3931
3932	assert(dst->bs == bs);
3933	if (src->bs->drv->bdrv_co_copy_range_to != raw_co_copy_range_to) {
3934	return -ENOTSUP;
3935	}
3936
3937	src_s = src->bs->opaque;
3938	if (fd_open(src->bs) < 0 \|\| fd_open(dst->bs) < 0) {
3939	return -EIO;
3940	}
3941
3942	acb = (RawPosixAIOData) {
3943	.bs = bs,
3944	.aio_type = QEMU_AIO_COPY_RANGE,
3945	.aio_fildes = src_s->fd,
3946	.aio_offset = src_offset,
3947	.aio_nbytes = bytes,
3948	.copy_range = {
3949	.aio_fd2 = s->fd,
3950	.aio_offset2 = dst_offset,
3951	},
3952	};
3953
3954	return raw_thread_pool_submit(handle_aiocb_copy_range, &acb);
3955	}
3956
3957	BlockDriver bdrv_file = {
3958	.format_name = "file",
3959	.protocol_name = "file",
3960	.instance_size = sizeof(BDRVRawState),
3961	.bdrv_needs_filename = true,
3962	.bdrv_probe = NULL, /* no probe for protocols */
3963	.bdrv_parse_filename = raw_parse_filename,
3964	.bdrv_open = raw_open,
3965	.bdrv_reopen_prepare = raw_reopen_prepare,
3966	.bdrv_reopen_commit = raw_reopen_commit,
3967	.bdrv_reopen_abort = raw_reopen_abort,
3968	.bdrv_close = raw_close,
3969	.bdrv_co_create = raw_co_create,
3970	.bdrv_co_create_opts = raw_co_create_opts,
3971	.bdrv_has_zero_init = bdrv_has_zero_init_1,
3972	.bdrv_co_block_status = raw_co_block_status,
3973	.bdrv_co_invalidate_cache = raw_co_invalidate_cache,
3974	.bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes,
3975	.bdrv_co_delete_file = raw_co_delete_file,
3976
3977	.bdrv_co_preadv = raw_co_preadv,
3978	.bdrv_co_pwritev = raw_co_pwritev,
3979	.bdrv_co_flush_to_disk = raw_co_flush_to_disk,
3980	.bdrv_co_pdiscard = raw_co_pdiscard,
3981	.bdrv_co_copy_range_from = raw_co_copy_range_from,
3982	.bdrv_co_copy_range_to = raw_co_copy_range_to,
3983	.bdrv_refresh_limits = raw_refresh_limits,
3984
3985	.bdrv_co_truncate = raw_co_truncate,
3986	.bdrv_co_getlength = raw_co_getlength,
3987	.bdrv_co_get_info = raw_co_get_info,
3988	.bdrv_get_specific_info = raw_get_specific_info,
3989	.bdrv_co_get_allocated_file_size = raw_co_get_allocated_file_size,
3990	.bdrv_get_specific_stats = raw_get_specific_stats,
3991	.bdrv_check_perm = raw_check_perm,
3992	.bdrv_set_perm = raw_set_perm,
3993	.bdrv_abort_perm_update = raw_abort_perm_update,
3994	.create_opts = &raw_create_opts,
3995	.mutable_opts = mutable_opts,
3996	};
3997
3998	/***********************************************/
3999	/* host device */
4000
4001	#if defined(HAVE_HOST_BLOCK_DEVICE)
4002
4003	#if defined(__APPLE__) && defined(__MACH__)
4004	static kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
4005	CFIndex maxPathSize, int flags);
4006
4007	static char FindEjectableOpticalMedia(io_iterator_t mediaIterator)
4008	{
4009	kern_return_t kernResult = KERN_FAILURE;
4010	mach_port_t mainPort;
4011	CFMutableDictionaryRef classesToMatch;
4012	const char *matching_array[] = {kIODVDMediaClass, kIOCDMediaClass};
4013	char *mediaType = NULL;
4014
4015	kernResult = IOMainPort(MACH_PORT_NULL, &mainPort);
4016	if ( KERN_SUCCESS != kernResult ) {
4017	printf("IOMainPort returned %d\n", kernResult);
4018	}
4019
4020	int index;
4021	for (index = 0; index < ARRAY_SIZE(matching_array); index++) {
4022	classesToMatch = IOServiceMatching(matching_array[index]);
4023	if (classesToMatch == NULL) {
4024	error_report("IOServiceMatching returned NULL for %s",
4025	matching_array[index]);
4026	continue;
4027	}
4028	CFDictionarySetValue(classesToMatch, CFSTR(kIOMediaEjectableKey),
4029	kCFBooleanTrue);
4030	kernResult = IOServiceGetMatchingServices(mainPort, classesToMatch,
4031	mediaIterator);
4032	if (kernResult != KERN_SUCCESS) {
4033	error_report("Note: IOServiceGetMatchingServices returned %d",
4034	kernResult);
4035	continue;
4036	}
4037
4038	/* If a match was found, leave the loop */
4039	if (*mediaIterator != 0) {
4040	trace_file_FindEjectableOpticalMedia(matching_array[index]);
4041	mediaType = g_strdup(matching_array[index]);
4042	break;
4043	}
4044	}
4045	return mediaType;
4046	}
4047
4048	kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
4049	CFIndex maxPathSize, int flags)
4050	{
4051	io_object_t nextMedia;
4052	kern_return_t kernResult = KERN_FAILURE;
4053	*bsdPath = '\0';
4054	nextMedia = IOIteratorNext( mediaIterator );
4055	if ( nextMedia )
4056	{
4057	CFTypeRef bsdPathAsCFString;
4058	bsdPathAsCFString = IORegistryEntryCreateCFProperty( nextMedia, CFSTR( kIOBSDNameKey ), kCFAllocatorDefault, 0 );
4059	if ( bsdPathAsCFString ) {
4060	size_t devPathLength;
4061	strcpy( bsdPath, _PATH_DEV );
4062	if (flags & BDRV_O_NOCACHE) {
4063	strcat(bsdPath, "r");
4064	}
4065	devPathLength = strlen( bsdPath );
4066	if ( CFStringGetCString( bsdPathAsCFString, bsdPath + devPathLength, maxPathSize - devPathLength, kCFStringEncodingASCII ) ) {
4067	kernResult = KERN_SUCCESS;
4068	}
4069	CFRelease( bsdPathAsCFString );
4070	}
4071	IOObjectRelease( nextMedia );
4072	}
4073
4074	return kernResult;
4075	}
4076
4077	/* Sets up a real cdrom for use in QEMU */
4078	static bool setup_cdrom(char bsd_path, Error *errp)
4079	{
4080	int index, num_of_test_partitions = 2, fd;
4081	char test_partition[MAXPATHLEN];
4082	bool partition_found = false;
4083
4084	/* look for a working partition */
4085	for (index = 0; index < num_of_test_partitions; index++) {
4086	snprintf(test_partition, sizeof(test_partition), "%ss%d", bsd_path,
4087	index);
4088	fd = qemu_open(test_partition, O_RDONLY \| O_BINARY \| O_LARGEFILE, NULL);
4089	if (fd >= 0) {
4090	partition_found = true;
4091	qemu_close(fd);
4092	break;
4093	}
4094	}
4095
4096	/* if a working partition on the device was not found */
4097	if (partition_found == false) {
4098	error_setg(errp, "Failed to find a working partition on disc");
4099	} else {
4100	trace_file_setup_cdrom(test_partition);
4101	pstrcpy(bsd_path, MAXPATHLEN, test_partition);
4102	}
4103	return partition_found;
4104	}
4105
4106	/* Prints directions on mounting and unmounting a device */
4107	static void print_unmounting_directions(const char *file_name)
4108	{
4109	error_report("If device %s is mounted on the desktop, unmount"
4110	" it first before using it in QEMU", file_name);
4111	error_report("Command to unmount device: diskutil unmountDisk %s",
4112	file_name);
4113	error_report("Command to mount device: diskutil mountDisk %s", file_name);
4114	}
4115
4116	#endif /* defined(__APPLE__) && defined(__MACH__) */
4117
4118	static int hdev_probe_device(const char *filename)
4119	{
4120	struct stat st;
4121
4122	/* allow a dedicated CD-ROM driver to match with a higher priority */
4123	if (strstart(filename, "/dev/cdrom", NULL))
4124	return 50;
4125
4126	if (stat(filename, &st) >= 0 &&
4127	(S_ISCHR(st.st_mode) \|\| S_ISBLK(st.st_mode))) {
4128	return 100;
4129	}
4130
4131	return 0;
4132	}
4133
4134	static void hdev_parse_filename(const char filename, QDict options,
4135	Error **errp)
4136	{
4137	bdrv_parse_filename_strip_prefix(filename, "host_device:", options);
4138	}
4139
4140	static bool hdev_is_sg(BlockDriverState *bs)
4141	{
4142
4143	#if defined(__linux__)
4144
4145	BDRVRawState *s = bs->opaque;
4146	struct stat st;
4147	struct sg_scsi_id scsiid;
4148	int sg_version;
4149	int ret;
4150
4151	if (stat(bs->filename, &st) < 0 \|\| !S_ISCHR(st.st_mode)) {
4152	return false;
4153	}
4154
4155	ret = ioctl(s->fd, SG_GET_VERSION_NUM, &sg_version);
4156	if (ret < 0) {
4157	return false;
4158	}
4159
4160	ret = ioctl(s->fd, SG_GET_SCSI_ID, &scsiid);
4161	if (ret >= 0) {
4162	trace_file_hdev_is_sg(scsiid.scsi_type, sg_version);
4163	return true;
4164	}
4165
4166	#endif
4167
4168	return false;
4169	}
4170
4171	static int hdev_open(BlockDriverState bs, QDict options, int flags,
4172	Error **errp)
4173	{
4174	BDRVRawState *s = bs->opaque;
4175	int ret;
4176
4177	#if defined(__APPLE__) && defined(__MACH__)
4178	/*
4179	* Caution: while qdict_get_str() is fine, getting non-string types
4180	* would require more care. When @options come from -blockdev or
4181	* blockdev_add, its members are typed according to the QAPI
4182	* schema, but when they come from -drive, they're all QString.
4183	*/
4184	const char *filename = qdict_get_str(options, "filename");
4185	char bsd_path[MAXPATHLEN] = "";
4186	bool error_occurred = false;
4187
4188	/* If using a real cdrom */
4189	if (strcmp(filename, "/dev/cdrom") == 0) {
4190	char *mediaType = NULL;
4191	kern_return_t ret_val;
4192	io_iterator_t mediaIterator = 0;
4193
4194	mediaType = FindEjectableOpticalMedia(&mediaIterator);
4195	if (mediaType == NULL) {
4196	error_setg(errp, "Please make sure your CD/DVD is in the optical"
4197	" drive");
4198	error_occurred = true;
4199	goto hdev_open_Mac_error;
4200	}
4201
4202	ret_val = GetBSDPath(mediaIterator, bsd_path, sizeof(bsd_path), flags);
4203	if (ret_val != KERN_SUCCESS) {
4204	error_setg(errp, "Could not get BSD path for optical drive");
4205	error_occurred = true;
4206	goto hdev_open_Mac_error;
4207	}
4208
4209	/* If a real optical drive was not found */
4210	if (bsd_path[0] == '\0') {
4211	error_setg(errp, "Failed to obtain bsd path for optical drive");
4212	error_occurred = true;
4213	goto hdev_open_Mac_error;
4214	}
4215
4216	/* If using a cdrom disc and finding a partition on the disc failed */
4217	if (strncmp(mediaType, kIOCDMediaClass, 9) == 0 &&
4218	setup_cdrom(bsd_path, errp) == false) {
4219	print_unmounting_directions(bsd_path);
4220	error_occurred = true;
4221	goto hdev_open_Mac_error;
4222	}
4223
4224	qdict_put_str(options, "filename", bsd_path);
4225
4226	hdev_open_Mac_error:
4227	g_free(mediaType);
4228	if (mediaIterator) {
4229	IOObjectRelease(mediaIterator);
4230	}
4231	if (error_occurred) {
4232	return -ENOENT;
4233	}
4234	}
4235	#endif /* defined(__APPLE__) && defined(__MACH__) */
4236
4237	s->type = FTYPE_FILE;
4238
4239	ret = raw_open_common(bs, options, flags, 0, true, errp);
4240	if (ret < 0) {
4241	#if defined(__APPLE__) && defined(__MACH__)
4242	if (*bsd_path) {
4243	filename = bsd_path;
4244	}
4245	/* if a physical device experienced an error while being opened */
4246	if (strncmp(filename, "/dev/", 5) == 0) {
4247	print_unmounting_directions(filename);
4248	}
4249	#endif /* defined(__APPLE__) && defined(__MACH__) */
4250	return ret;
4251	}
4252
4253	/* Since this does ioctl the device must be already opened */
4254	bs->sg = hdev_is_sg(bs);
4255
4256	/* sg devices aren't even block devices and can't use dm-mpath */
4257	s->use_mpath = !bs->sg;
4258
4259	return ret;
4260	}
4261
4262	#if defined(__linux__)
4263	#if defined(DM_MPATH_PROBE_PATHS)
4264	static bool coroutine_fn sgio_path_error(int ret, sg_io_hdr_t *io_hdr)
4265	{
4266	if (ret < 0) {
4267	/* Path errors sometimes result in -ENODEV */
4268	return ret == -ENODEV;
4269	}
4270
4271	if (io_hdr->host_status != SCSI_HOST_OK) {
4272	return true;
4273	}
4274
4275	switch (io_hdr->status) {
4276	case GOOD:
4277	case CONDITION_GOOD:
4278	case INTERMEDIATE_GOOD:
4279	case INTERMEDIATE_C_GOOD:
4280	case RESERVATION_CONFLICT:
4281	case COMMAND_TERMINATED:
4282	return false;
4283	case CHECK_CONDITION:
4284	return !scsi_sense_buf_is_guest_recoverable(io_hdr->sbp,
4285	io_hdr->mx_sb_len);
4286	default:
4287	return true;
4288	}
4289	}
4290
4291	static bool coroutine_fn hdev_co_ioctl_sgio_retry(RawPosixAIOData *acb, int ret)
4292	{
4293	BDRVRawState *s = acb->bs->opaque;
4294	RawPosixAIOData probe_acb;
4295
4296	if (!s->use_mpath) {
4297	return false;
4298	}
4299
4300	if (!sgio_path_error(ret, acb->ioctl.buf)) {
4301	return false;
4302	}
4303
4304	probe_acb = (RawPosixAIOData) {
4305	.bs = acb->bs,
4306	.aio_type = QEMU_AIO_IOCTL,
4307	.aio_fildes = s->fd,
4308	.aio_offset = 0,
4309	.ioctl = {
4310	.buf = NULL,
4311	.cmd = DM_MPATH_PROBE_PATHS,
4312	},
4313	};
4314
4315	ret = raw_thread_pool_submit(handle_aiocb_ioctl, &probe_acb);
4316	if (ret == -ENOTTY) {
4317	s->use_mpath = false;
4318	} else if (ret == -EAGAIN) {
4319	/* The device might be suspended for a table reload, worth retrying */
4320	return true;
4321	}
4322
4323	return ret == 0;
4324	}
4325	#else
4326	static bool coroutine_fn hdev_co_ioctl_sgio_retry(RawPosixAIOData *acb, int ret)
4327	{
4328	return false;
4329	}
4330	#endif /* DM_MPATH_PROBE_PATHS */
4331
4332	static int coroutine_fn
4333	hdev_co_ioctl(BlockDriverState bs, unsigned long int req, void buf)
4334	{
4335	BDRVRawState *s = bs->opaque;
4336	RawPosixAIOData acb;
4337	uint64_t eagain_sleep_ns = 1 * SCALE_MS;
4338	int retries = SG_IO_MAX_RETRIES;
4339	int ret;
4340
4341	ret = fd_open(bs);
4342	if (ret < 0) {
4343	return ret;
4344	}
4345
4346	if (req == SG_IO && s->pr_mgr) {
4347	struct sg_io_hdr *io_hdr = buf;
4348	if (io_hdr->cmdp[0] == PERSISTENT_RESERVE_OUT \|\|
4349	io_hdr->cmdp[0] == PERSISTENT_RESERVE_IN) {
4350	return pr_manager_execute(s->pr_mgr, qemu_get_current_aio_context(),
4351	s->fd, io_hdr);
4352	}
4353	}
4354
4355	acb = (RawPosixAIOData) {
4356	.bs = bs,
4357	.aio_type = QEMU_AIO_IOCTL,
4358	.aio_fildes = s->fd,
4359	.aio_offset = 0,
4360	.ioctl = {
4361	.buf = buf,
4362	.cmd = req,
4363	},
4364	};
4365
4366	retry:
4367	ret = raw_thread_pool_submit(handle_aiocb_ioctl, &acb);
4368	if (req == SG_IO && s->use_mpath) {
4369	if (ret == -EAGAIN && eagain_sleep_ns < NANOSECONDS_PER_SECOND) {
4370	/*
4371	* If this is a multipath device, it is probably suspended.
4372	*
4373	* This can happen while the dm table is reloaded, e.g. because a
4374	* path is added or removed. This is an operation that should
4375	* complete within 1ms, so just wait a bit and retry.
4376	*
4377	* There are also some cases in which libmpathpersist must recover
4378	* from path failure during its operation, which can leave the
4379	* device suspended for a bit longer while the library brings back
4380	* reservations into the expected state.
4381	*
4382	* Use increasing delays to cover both cases without waiting
4383	* excessively, and stop after a bit more than a second (1023 ms).
4384	* This is a tolerable delay before we return an error and
4385	* potentially stop the VM.
4386	*/
4387	qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, eagain_sleep_ns);
4388	eagain_sleep_ns *= 2;
4389	goto retry;
4390	}
4391
4392	/* Even for ret == 0, the SG_IO header can contain an error */
4393	if (retries-- && hdev_co_ioctl_sgio_retry(&acb, ret)) {
4394	goto retry;
4395	}
4396	}
4397
4398	return ret;
4399	}
4400	#endif /* linux */
4401
4402	static coroutine_fn int
4403	hdev_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
4404	{
4405	BDRVRawState *s = bs->opaque;
4406	int ret;
4407
4408	ret = fd_open(bs);
4409	if (ret < 0) {
4410	raw_account_discard(s, bytes, ret);
4411	return ret;
4412	}
4413	return raw_do_pdiscard(bs, offset, bytes, true);
4414	}
4415
4416	static coroutine_fn int hdev_co_pwrite_zeroes(BlockDriverState *bs,
4417	int64_t offset, int64_t bytes, BdrvRequestFlags flags)
4418	{
4419	int rc;
4420
4421	rc = fd_open(bs);
4422	if (rc < 0) {
4423	return rc;
4424	}
4425
4426	return raw_do_pwrite_zeroes(bs, offset, bytes, flags, true);
4427	}
4428
4429	static BlockDriver bdrv_host_device = {
4430	.format_name = "host_device",
4431	.protocol_name = "host_device",
4432	.instance_size = sizeof(BDRVRawState),
4433	.bdrv_needs_filename = true,
4434	.bdrv_probe_device = hdev_probe_device,
4435	.bdrv_parse_filename = hdev_parse_filename,
4436	.bdrv_open = hdev_open,
4437	.bdrv_close = raw_close,
4438	.bdrv_reopen_prepare = raw_reopen_prepare,
4439	.bdrv_reopen_commit = raw_reopen_commit,
4440	.bdrv_reopen_abort = raw_reopen_abort,
4441	.bdrv_co_create_opts = bdrv_co_create_opts_simple,
4442	.create_opts = &bdrv_create_opts_simple,
4443	.mutable_opts = mutable_opts,
4444	.bdrv_co_invalidate_cache = raw_co_invalidate_cache,
4445	.bdrv_co_pwrite_zeroes = hdev_co_pwrite_zeroes,
4446
4447	.bdrv_co_preadv = raw_co_preadv,
4448	.bdrv_co_pwritev = raw_co_pwritev,
4449	.bdrv_co_flush_to_disk = raw_co_flush_to_disk,
4450	.bdrv_co_pdiscard = hdev_co_pdiscard,
4451	.bdrv_co_copy_range_from = raw_co_copy_range_from,
4452	.bdrv_co_copy_range_to = raw_co_copy_range_to,
4453	.bdrv_refresh_limits = raw_refresh_limits,
4454
4455	.bdrv_co_truncate = raw_co_truncate,
4456	.bdrv_co_getlength = raw_co_getlength,
4457	.bdrv_co_get_info = raw_co_get_info,
4458	.bdrv_get_specific_info = raw_get_specific_info,
4459	.bdrv_co_get_allocated_file_size = raw_co_get_allocated_file_size,
4460	.bdrv_get_specific_stats = hdev_get_specific_stats,
4461	.bdrv_check_perm = raw_check_perm,
4462	.bdrv_set_perm = raw_set_perm,
4463	.bdrv_abort_perm_update = raw_abort_perm_update,
4464	.bdrv_probe_blocksizes = hdev_probe_blocksizes,
4465	.bdrv_probe_geometry = hdev_probe_geometry,
4466
4467	/* generic scsi device */
4468	#ifdef __linux__
4469	.bdrv_co_ioctl = hdev_co_ioctl,
4470	#endif
4471
4472	/* zoned device */
4473	#if defined(CONFIG_BLKZONED)
4474	/* zone management operations */
4475	.bdrv_co_zone_report = raw_co_zone_report,
4476	.bdrv_co_zone_mgmt = raw_co_zone_mgmt,
4477	.bdrv_co_zone_append = raw_co_zone_append,
4478	#endif
4479	};
4480
4481	#if defined(__linux__) \|\| defined(__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
4482	static void cdrom_parse_filename(const char filename, QDict options,
4483	Error **errp)
4484	{
4485	bdrv_parse_filename_strip_prefix(filename, "host_cdrom:", options);
4486	}
4487
4488	static void cdrom_refresh_limits(BlockDriverState bs, Error *errp)
4489	{
4490	bs->bl.has_variable_length = true;
4491	raw_refresh_limits(bs, errp);
4492	}
4493	#endif
4494
4495	#ifdef __linux__
4496	static int cdrom_open(BlockDriverState bs, QDict options, int flags,
4497	Error **errp)
4498	{
4499	BDRVRawState *s = bs->opaque;
4500
4501	s->type = FTYPE_CD;
4502
4503	/* open will not fail even if no CD is inserted, so add O_NONBLOCK */
4504	return raw_open_common(bs, options, flags, O_NONBLOCK, true, errp);
4505	}
4506
4507	static int cdrom_probe_device(const char *filename)
4508	{
4509	int fd, ret;
4510	int prio = 0;
4511	struct stat st;
4512
4513	fd = qemu_open(filename, O_RDONLY \| O_NONBLOCK, NULL);
4514	if (fd < 0) {
4515	goto out;
4516	}
4517	ret = fstat(fd, &st);
4518	if (ret == -1 \|\| !S_ISBLK(st.st_mode)) {
4519	goto outc;
4520	}
4521
4522	/* Attempt to detect via a CDROM specific ioctl */
4523	ret = ioctl(fd, CDROM_DRIVE_STATUS, CDSL_CURRENT);
4524	if (ret >= 0)
4525	prio = 100;
4526
4527	outc:
4528	qemu_close(fd);
4529	out:
4530	return prio;
4531	}
4532
4533	static bool coroutine_fn cdrom_co_is_inserted(BlockDriverState *bs)
4534	{
4535	BDRVRawState *s = bs->opaque;
4536	int ret;
4537
4538	ret = ioctl(s->fd, CDROM_DRIVE_STATUS, CDSL_CURRENT);
4539	return ret == CDS_DISC_OK;
4540	}
4541
4542	static void coroutine_fn cdrom_co_eject(BlockDriverState *bs, bool eject_flag)
4543	{
4544	BDRVRawState *s = bs->opaque;
4545
4546	if (eject_flag) {
4547	if (ioctl(s->fd, CDROMEJECT, NULL) < 0)
4548	perror("CDROMEJECT");
4549	} else {
4550	if (ioctl(s->fd, CDROMCLOSETRAY, NULL) < 0)
4551	perror("CDROMEJECT");
4552	}
4553	}
4554
4555	static void coroutine_fn cdrom_co_lock_medium(BlockDriverState *bs, bool locked)
4556	{
4557	BDRVRawState *s = bs->opaque;
4558
4559	if (ioctl(s->fd, CDROM_LOCKDOOR, locked) < 0) {
4560	/*
4561	* Note: an error can happen if the distribution automatically
4562	* mounts the CD-ROM
4563	*/
4564	/* perror("CDROM_LOCKDOOR"); */
4565	}
4566	}
4567
4568	static BlockDriver bdrv_host_cdrom = {
4569	.format_name = "host_cdrom",
4570	.protocol_name = "host_cdrom",
4571	.instance_size = sizeof(BDRVRawState),
4572	.bdrv_needs_filename = true,
4573	.bdrv_probe_device = cdrom_probe_device,
4574	.bdrv_parse_filename = cdrom_parse_filename,
4575	.bdrv_open = cdrom_open,
4576	.bdrv_close = raw_close,
4577	.bdrv_reopen_prepare = raw_reopen_prepare,
4578	.bdrv_reopen_commit = raw_reopen_commit,
4579	.bdrv_reopen_abort = raw_reopen_abort,
4580	.bdrv_co_create_opts = bdrv_co_create_opts_simple,
4581	.create_opts = &bdrv_create_opts_simple,
4582	.mutable_opts = mutable_opts,
4583	.bdrv_co_invalidate_cache = raw_co_invalidate_cache,
4584
4585	.bdrv_co_preadv = raw_co_preadv,
4586	.bdrv_co_pwritev = raw_co_pwritev,
4587	.bdrv_co_flush_to_disk = raw_co_flush_to_disk,
4588	.bdrv_refresh_limits = cdrom_refresh_limits,
4589
4590	.bdrv_co_truncate = raw_co_truncate,
4591	.bdrv_co_getlength = raw_co_getlength,
4592	.bdrv_co_get_allocated_file_size = raw_co_get_allocated_file_size,
4593
4594	/* removable device support */
4595	.bdrv_co_is_inserted = cdrom_co_is_inserted,
4596	.bdrv_co_eject = cdrom_co_eject,
4597	.bdrv_co_lock_medium = cdrom_co_lock_medium,
4598
4599	/* generic scsi device */
4600	.bdrv_co_ioctl = hdev_co_ioctl,
4601	};
4602	#endif /* __linux__ */
4603
4604	#if defined (__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
4605	static int cdrom_open(BlockDriverState bs, QDict options, int flags,
4606	Error **errp)
4607	{
4608	BDRVRawState *s = bs->opaque;
4609	int ret;
4610
4611	s->type = FTYPE_CD;
4612
4613	ret = raw_open_common(bs, options, flags, 0, true, errp);
4614	if (ret) {
4615	return ret;
4616	}
4617
4618	/* make sure the door isn't locked at this time */
4619	ioctl(s->fd, CDIOCALLOW);
4620	return 0;
4621	}
4622
4623	static int cdrom_probe_device(const char *filename)
4624	{
4625	if (strstart(filename, "/dev/cd", NULL) \|\|
4626	strstart(filename, "/dev/acd", NULL))
4627	return 100;
4628	return 0;
4629	}
4630
4631	static int cdrom_reopen(BlockDriverState *bs)
4632	{
4633	BDRVRawState *s = bs->opaque;
4634	int fd;
4635
4636	/*
4637	* Force reread of possibly changed/newly loaded disc,
4638	* FreeBSD seems to not notice sometimes...
4639	*/
4640	if (s->fd >= 0)
4641	qemu_close(s->fd);
4642	fd = qemu_open(bs->filename, s->open_flags, NULL);
4643	if (fd < 0) {
4644	s->fd = -1;
4645	return -EIO;
4646	}
4647	s->fd = fd;
4648
4649	/* make sure the door isn't locked at this time */
4650	ioctl(s->fd, CDIOCALLOW);
4651	return 0;
4652	}
4653
4654	static bool coroutine_fn cdrom_co_is_inserted(BlockDriverState *bs)
4655	{
4656	return raw_getlength(bs) > 0;
4657	}
4658
4659	static void coroutine_fn cdrom_co_eject(BlockDriverState *bs, bool eject_flag)
4660	{
4661	BDRVRawState *s = bs->opaque;
4662
4663	if (s->fd < 0)
4664	return;
4665
4666	(void) ioctl(s->fd, CDIOCALLOW);
4667
4668	if (eject_flag) {
4669	if (ioctl(s->fd, CDIOCEJECT) < 0)
4670	perror("CDIOCEJECT");
4671	} else {
4672	if (ioctl(s->fd, CDIOCCLOSE) < 0)
4673	perror("CDIOCCLOSE");
4674	}
4675
4676	cdrom_reopen(bs);
4677	}
4678
4679	static void coroutine_fn cdrom_co_lock_medium(BlockDriverState *bs, bool locked)
4680	{
4681	BDRVRawState *s = bs->opaque;
4682
4683	if (s->fd < 0)
4684	return;
4685	if (ioctl(s->fd, (locked ? CDIOCPREVENT : CDIOCALLOW)) < 0) {
4686	/*
4687	* Note: an error can happen if the distribution automatically
4688	* mounts the CD-ROM
4689	*/
4690	/* perror("CDROM_LOCKDOOR"); */
4691	}
4692	}
4693
4694	static BlockDriver bdrv_host_cdrom = {
4695	.format_name = "host_cdrom",
4696	.protocol_name = "host_cdrom",
4697	.instance_size = sizeof(BDRVRawState),
4698	.bdrv_needs_filename = true,
4699	.bdrv_probe_device = cdrom_probe_device,
4700	.bdrv_parse_filename = cdrom_parse_filename,
4701	.bdrv_open = cdrom_open,
4702	.bdrv_close = raw_close,
4703	.bdrv_reopen_prepare = raw_reopen_prepare,
4704	.bdrv_reopen_commit = raw_reopen_commit,
4705	.bdrv_reopen_abort = raw_reopen_abort,
4706	.bdrv_co_create_opts = bdrv_co_create_opts_simple,
4707	.create_opts = &bdrv_create_opts_simple,
4708	.mutable_opts = mutable_opts,
4709
4710	.bdrv_co_preadv = raw_co_preadv,
4711	.bdrv_co_pwritev = raw_co_pwritev,
4712	.bdrv_co_flush_to_disk = raw_co_flush_to_disk,
4713	.bdrv_refresh_limits = cdrom_refresh_limits,
4714
4715	.bdrv_co_truncate = raw_co_truncate,
4716	.bdrv_co_getlength = raw_co_getlength,
4717	.bdrv_co_get_allocated_file_size = raw_co_get_allocated_file_size,
4718
4719	/* removable device support */
4720	.bdrv_co_is_inserted = cdrom_co_is_inserted,
4721	.bdrv_co_eject = cdrom_co_eject,
4722	.bdrv_co_lock_medium = cdrom_co_lock_medium,
4723	};
4724	#endif /* __FreeBSD__ */
4725
4726	#endif /* HAVE_HOST_BLOCK_DEVICE */
4727
4728	static void bdrv_file_init(void)
4729	{
4730	/*
4731	* Register all the drivers. Note that order is important, the driver
4732	* registered last will get probed first.
4733	*/
4734	bdrv_register(&bdrv_file);
4735	#if defined(HAVE_HOST_BLOCK_DEVICE)
4736	bdrv_register(&bdrv_host_device);
4737	#ifdef __linux__
4738	bdrv_register(&bdrv_host_cdrom);
4739	#endif
4740	#if defined(__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
4741	bdrv_register(&bdrv_host_cdrom);
4742	#endif
4743	#endif /* HAVE_HOST_BLOCK_DEVICE */
4744	}
4745
4746	block_init(bdrv_file_init);