qcow2-cluster.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,604 lines 88.3 KB

Raw

1	/*
2	* Block driver for the QCOW version 2 format
3	*
4	* Copyright (c) 2004-2006 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	#include "qemu/osdep.h"
26	#include <zlib.h>
27
28	#include "block/block-io.h"
29	#include "qapi/error.h"
30	#include "qcow2.h"
31	#include "qemu/bswap.h"
32	#include "qemu/memalign.h"
33	#include "trace.h"
34
35	int coroutine_fn qcow2_shrink_l1_table(BlockDriverState *bs,
36	uint64_t exact_size)
37	{
38	BDRVQcow2State *s = bs->opaque;
39	int new_l1_size, i, ret;
40
41	if (exact_size >= s->l1_size) {
42	return 0;
43	}
44
45	new_l1_size = exact_size;
46
47	#ifdef DEBUG_ALLOC2
48	fprintf(stderr, "shrink l1_table from %d to %d\n", s->l1_size, new_l1_size);
49	#endif
50
51	BLKDBG_CO_EVENT(bs->file, BLKDBG_L1_SHRINK_WRITE_TABLE);
52	ret = bdrv_co_pwrite_zeroes(bs->file,
53	s->l1_table_offset + new_l1_size * L1E_SIZE,
54	(s->l1_size - new_l1_size) * L1E_SIZE, 0);
55	if (ret < 0) {
56	goto fail;
57	}
58
59	ret = bdrv_co_flush(bs->file->bs);
60	if (ret < 0) {
61	goto fail;
62	}
63
64	BLKDBG_CO_EVENT(bs->file, BLKDBG_L1_SHRINK_FREE_L2_CLUSTERS);
65	for (i = s->l1_size - 1; i > new_l1_size - 1; i--) {
66	if ((s->l1_table[i] & L1E_OFFSET_MASK) == 0) {
67	continue;
68	}
69	qcow2_free_clusters(bs, s->l1_table[i] & L1E_OFFSET_MASK,
70	s->cluster_size, QCOW2_DISCARD_ALWAYS);
71	s->l1_table[i] = 0;
72	}
73	return 0;
74
75	fail:
76	/*
77	* If the write in the l1_table failed the image may contain a partially
78	* overwritten l1_table. In this case it would be better to clear the
79	* l1_table in memory to avoid possible image corruption.
80	*/
81	memset(s->l1_table + new_l1_size, 0,
82	(s->l1_size - new_l1_size) * L1E_SIZE);
83	return ret;
84	}
85
86	int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
87	bool exact_size)
88	{
89	BDRVQcow2State *s = bs->opaque;
90	int new_l1_size2, ret, i;
91	uint64_t *new_l1_table;
92	int64_t old_l1_table_offset, old_l1_size;
93	int64_t new_l1_table_offset, new_l1_size;
94	uint8_t data[12];
95
96	if (min_size <= s->l1_size)
97	return 0;
98
99	/* Do a sanity check on min_size before trying to calculate new_l1_size
100	* (this prevents overflows during the while loop for the calculation of
101	* new_l1_size) */
102	if (min_size > INT_MAX / L1E_SIZE) {
103	return -EFBIG;
104	}
105
106	if (exact_size) {
107	new_l1_size = min_size;
108	} else {
109	/* Bump size up to reduce the number of times we have to grow */
110	new_l1_size = s->l1_size;
111	if (new_l1_size == 0) {
112	new_l1_size = 1;
113	}
114	while (min_size > new_l1_size) {
115	new_l1_size = DIV_ROUND_UP(new_l1_size * 3, 2);
116	}
117	}
118
119	QEMU_BUILD_BUG_ON(QCOW_MAX_L1_SIZE > INT_MAX);
120	if (new_l1_size > QCOW_MAX_L1_SIZE / L1E_SIZE) {
121	return -EFBIG;
122	}
123
124	#ifdef DEBUG_ALLOC2
125	fprintf(stderr, "grow l1_table from %d to %" PRId64 "\n",
126	s->l1_size, new_l1_size);
127	#endif
128
129	new_l1_size2 = L1E_SIZE * new_l1_size;
130	new_l1_table = qemu_try_blockalign(bs->file->bs, new_l1_size2);
131	if (new_l1_table == NULL) {
132	return -ENOMEM;
133	}
134	memset(new_l1_table, 0, new_l1_size2);
135
136	if (s->l1_size) {
137	memcpy(new_l1_table, s->l1_table, s->l1_size * L1E_SIZE);
138	}
139
140	/* write new table (align to cluster) */
141	BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ALLOC_TABLE);
142	new_l1_table_offset = qcow2_alloc_clusters(bs, new_l1_size2);
143	if (new_l1_table_offset < 0) {
144	qemu_vfree(new_l1_table);
145	return new_l1_table_offset;
146	}
147
148	ret = qcow2_cache_flush(bs, s->refcount_block_cache);
149	if (ret < 0) {
150	goto fail;
151	}
152
153	/* the L1 position has not yet been updated, so these clusters must
154	* indeed be completely free */
155	ret = qcow2_pre_write_overlap_check(bs, 0, new_l1_table_offset,
156	new_l1_size2, false);
157	if (ret < 0) {
158	goto fail;
159	}
160
161	BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
162	for(i = 0; i < s->l1_size; i++)
163	new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
164	ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_size2,
165	new_l1_table, 0);
166	if (ret < 0)
167	goto fail;
168	for(i = 0; i < s->l1_size; i++)
169	new_l1_table[i] = be64_to_cpu(new_l1_table[i]);
170
171	/* set new table */
172	BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
173	stl_be_p(data, new_l1_size);
174	stq_be_p(data + 4, new_l1_table_offset);
175	ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size),
176	sizeof(data), data, 0);
177	if (ret < 0) {
178	goto fail;
179	}
180	qemu_vfree(s->l1_table);
181	old_l1_table_offset = s->l1_table_offset;
182	s->l1_table_offset = new_l1_table_offset;
183	s->l1_table = new_l1_table;
184	old_l1_size = s->l1_size;
185	s->l1_size = new_l1_size;
186	qcow2_free_clusters(bs, old_l1_table_offset, old_l1_size * L1E_SIZE,
187	QCOW2_DISCARD_OTHER);
188	return 0;
189	fail:
190	qemu_vfree(new_l1_table);
191	qcow2_free_clusters(bs, new_l1_table_offset, new_l1_size2,
192	QCOW2_DISCARD_OTHER);
193	return ret;
194	}
195
196	/*
197	* l2_load
198	*
199	* @bs: The BlockDriverState
200	* @offset: A guest offset, used to calculate what slice of the L2
201	* table to load.
202	* @l2_offset: Offset to the L2 table in the image file.
203	* @l2_slice: Location to store the pointer to the L2 slice.
204	*
205	* Loads a L2 slice into memory (L2 slices are the parts of L2 tables
206	* that are loaded by the qcow2 cache). If the slice is in the cache,
207	* the cache is used; otherwise the L2 slice is loaded from the image
208	* file.
209	*/
210	static int GRAPH_RDLOCK
211	l2_load(BlockDriverState *bs, uint64_t offset,
212	uint64_t l2_offset, uint64_t **l2_slice)
213	{
214	BDRVQcow2State *s = bs->opaque;
215	int start_of_slice = l2_entry_size(s) *
216	(offset_to_l2_index(s, offset) - offset_to_l2_slice_index(s, offset));
217
218	return qcow2_cache_get(bs, s->l2_table_cache, l2_offset + start_of_slice,
219	(void **)l2_slice);
220	}
221
222	/*
223	* Writes an L1 entry to disk (note that depending on the alignment
224	* requirements this function may write more that just one entry in
225	* order to prevent bdrv_pwrite from performing a read-modify-write)
226	*/
227	int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
228	{
229	BDRVQcow2State *s = bs->opaque;
230	int l1_start_index;
231	int i, ret;
232	int bufsize = MAX(L1E_SIZE,
233	MIN(bs->file->bs->bl.request_alignment, s->cluster_size));
234	int nentries = bufsize / L1E_SIZE;
235	g_autofree uint64_t *buf = g_try_new0(uint64_t, nentries);
236
237	if (buf == NULL) {
238	return -ENOMEM;
239	}
240
241	l1_start_index = QEMU_ALIGN_DOWN(l1_index, nentries);
242	for (i = 0; i < MIN(nentries, s->l1_size - l1_start_index); i++) {
243	buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
244	}
245
246	ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L1,
247	s->l1_table_offset + L1E_SIZE * l1_start_index, bufsize, false);
248	if (ret < 0) {
249	return ret;
250	}
251
252	BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
253	ret = bdrv_pwrite_sync(bs->file,
254	s->l1_table_offset + L1E_SIZE * l1_start_index,
255	bufsize, buf, 0);
256	if (ret < 0) {
257	return ret;
258	}
259
260	return 0;
261	}
262
263	/*
264	* l2_allocate
265	*
266	* Allocate a new l2 entry in the file. If l1_index points to an already
267	* used entry in the L2 table (i.e. we are doing a copy on write for the L2
268	* table) copy the contents of the old L2 table into the newly allocated one.
269	* Otherwise the new table is initialized with zeros.
270	*
271	*/
272
273	static int GRAPH_RDLOCK l2_allocate(BlockDriverState *bs, int l1_index)
274	{
275	BDRVQcow2State *s = bs->opaque;
276	uint64_t old_l2_offset;
277	uint64_t *l2_slice = NULL;
278	unsigned slice, slice_size2, n_slices;
279	int64_t l2_offset;
280	int ret;
281
282	old_l2_offset = s->l1_table[l1_index];
283
284	trace_qcow2_l2_allocate(bs, l1_index);
285
286	/* allocate a new l2 entry */
287
288	l2_offset = qcow2_alloc_clusters(bs, s->l2_size * l2_entry_size(s));
289	if (l2_offset < 0) {
290	ret = l2_offset;
291	goto fail;
292	}
293
294	/* The offset must fit in the offset field of the L1 table entry */
295	assert((l2_offset & L1E_OFFSET_MASK) == l2_offset);
296
297	/* If we're allocating the table at offset 0 then something is wrong */
298	if (l2_offset == 0) {
299	qcow2_signal_corruption(bs, true, -1, -1, "Preventing invalid "
300	"allocation of L2 table at offset 0");
301	ret = -EIO;
302	goto fail;
303	}
304
305	ret = qcow2_cache_flush(bs, s->refcount_block_cache);
306	if (ret < 0) {
307	goto fail;
308	}
309
310	/* allocate a new entry in the l2 cache */
311
312	slice_size2 = s->l2_slice_size * l2_entry_size(s);
313	n_slices = s->cluster_size / slice_size2;
314
315	trace_qcow2_l2_allocate_get_empty(bs, l1_index);
316	for (slice = 0; slice < n_slices; slice++) {
317	ret = qcow2_cache_get_empty(bs, s->l2_table_cache,
318	l2_offset + slice * slice_size2,
319	(void **) &l2_slice);
320	if (ret < 0) {
321	goto fail;
322	}
323
324	if ((old_l2_offset & L1E_OFFSET_MASK) == 0) {
325	/* if there was no old l2 table, clear the new slice */
326	memset(l2_slice, 0, slice_size2);
327	} else {
328	uint64_t *old_slice;
329	uint64_t old_l2_slice_offset =
330	(old_l2_offset & L1E_OFFSET_MASK) + slice * slice_size2;
331
332	/* if there was an old l2 table, read a slice from the disk */
333	BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_COW_READ);
334	ret = qcow2_cache_get(bs, s->l2_table_cache, old_l2_slice_offset,
335	(void **) &old_slice);
336	if (ret < 0) {
337	goto fail;
338	}
339
340	memcpy(l2_slice, old_slice, slice_size2);
341
342	qcow2_cache_put(s->l2_table_cache, (void **) &old_slice);
343	}
344
345	/* write the l2 slice to the file */
346	BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);
347
348	trace_qcow2_l2_allocate_write_l2(bs, l1_index);
349	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
350	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
351	}
352
353	ret = qcow2_cache_flush(bs, s->l2_table_cache);
354	if (ret < 0) {
355	goto fail;
356	}
357
358	/* update the L1 entry */
359	trace_qcow2_l2_allocate_write_l1(bs, l1_index);
360	s->l1_table[l1_index] = l2_offset \| QCOW_OFLAG_COPIED;
361	ret = qcow2_write_l1_entry(bs, l1_index);
362	if (ret < 0) {
363	goto fail;
364	}
365
366	trace_qcow2_l2_allocate_done(bs, l1_index, 0);
367	return 0;
368
369	fail:
370	trace_qcow2_l2_allocate_done(bs, l1_index, ret);
371	if (l2_slice != NULL) {
372	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
373	}
374	s->l1_table[l1_index] = old_l2_offset;
375	if (l2_offset > 0) {
376	qcow2_free_clusters(bs, l2_offset, s->l2_size * l2_entry_size(s),
377	QCOW2_DISCARD_ALWAYS);
378	}
379	return ret;
380	}
381
382	/*
383	* For a given L2 entry, count the number of contiguous subclusters of
384	* the same type starting from @sc_from. Compressed clusters are
385	* treated as if they were divided into subclusters of size
386	* s->subcluster_size.
387	*
388	* Return the number of contiguous subclusters and set @type to the
389	* subcluster type.
390	*
391	* If the L2 entry is invalid return -errno and set @type to
392	* QCOW2_SUBCLUSTER_INVALID.
393	*/
394	static int GRAPH_RDLOCK
395	qcow2_get_subcluster_range_type(BlockDriverState *bs, uint64_t l2_entry,
396	uint64_t l2_bitmap, unsigned sc_from,
397	QCow2SubclusterType *type)
398	{
399	BDRVQcow2State *s = bs->opaque;
400	uint32_t val;
401
402	*type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_from);
403
404	if (*type == QCOW2_SUBCLUSTER_INVALID) {
405	return -EINVAL;
406	} else if (!has_subclusters(s) \|\| *type == QCOW2_SUBCLUSTER_COMPRESSED) {
407	return s->subclusters_per_cluster - sc_from;
408	}
409
410	switch (*type) {
411	case QCOW2_SUBCLUSTER_NORMAL:
412	val = l2_bitmap \| QCOW_OFLAG_SUB_ALLOC_RANGE(0, sc_from);
413	return cto32(val) - sc_from;
414
415	case QCOW2_SUBCLUSTER_ZERO_PLAIN:
416	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
417	val = (l2_bitmap \| QCOW_OFLAG_SUB_ZERO_RANGE(0, sc_from)) >> 32;
418	return cto32(val) - sc_from;
419
420	case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
421	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
422	val = ((l2_bitmap >> 32) \| l2_bitmap)
423	& ~QCOW_OFLAG_SUB_ALLOC_RANGE(0, sc_from);
424	return ctz32(val) - sc_from;
425
426	default:
427	g_assert_not_reached();
428	}
429	}
430
431	/*
432	* Return the number of contiguous subclusters of the exact same type
433	* in a given L2 slice, starting from cluster @l2_index, subcluster
434	* @sc_index. Allocated subclusters are required to be contiguous in
435	* the image file.
436	* At most @nb_clusters are checked (note that this means clusters,
437	* not subclusters).
438	* Compressed clusters are always processed one by one but for the
439	* purpose of this count they are treated as if they were divided into
440	* subclusters of size s->subcluster_size.
441	* On failure return -errno and update @l2_index to point to the
442	* invalid entry.
443	*/
444	static int GRAPH_RDLOCK
445	count_contiguous_subclusters(BlockDriverState *bs, int nb_clusters,
446	unsigned sc_index, uint64_t *l2_slice,
447	unsigned *l2_index)
448	{
449	BDRVQcow2State *s = bs->opaque;
450	int i, count = 0;
451	bool check_offset = false;
452	uint64_t expected_offset = 0;
453	QCow2SubclusterType expected_type = QCOW2_SUBCLUSTER_NORMAL, type;
454
455	assert(*l2_index + nb_clusters <= s->l2_slice_size);
456
457	for (i = 0; i < nb_clusters; i++) {
458	unsigned first_sc = (i == 0) ? sc_index : 0;
459	uint64_t l2_entry = get_l2_entry(s, l2_slice, *l2_index + i);
460	uint64_t l2_bitmap = get_l2_bitmap(s, l2_slice, *l2_index + i);
461	int ret = qcow2_get_subcluster_range_type(bs, l2_entry, l2_bitmap,
462	first_sc, &type);
463	if (ret < 0) {
464	l2_index += i; / Point to the invalid entry */
465	return -EIO;
466	}
467	if (i == 0) {
468	if (type == QCOW2_SUBCLUSTER_COMPRESSED) {
469	/* Compressed clusters are always processed one by one */
470	return ret;
471	}
472	expected_type = type;
473	expected_offset = l2_entry & L2E_OFFSET_MASK;
474	check_offset = (type == QCOW2_SUBCLUSTER_NORMAL \|\|
475	type == QCOW2_SUBCLUSTER_ZERO_ALLOC \|\|
476	type == QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC);
477	} else if (type != expected_type) {
478	break;
479	} else if (check_offset) {
480	expected_offset += s->cluster_size;
481	if (expected_offset != (l2_entry & L2E_OFFSET_MASK)) {
482	break;
483	}
484	}
485	count += ret;
486	/* Stop if there are type changes before the end of the cluster */
487	if (first_sc + ret < s->subclusters_per_cluster) {
488	break;
489	}
490	}
491
492	return count;
493	}
494
495	static int coroutine_fn GRAPH_RDLOCK
496	do_perform_cow_read(BlockDriverState *bs, uint64_t src_cluster_offset,
497	unsigned offset_in_cluster, QEMUIOVector *qiov)
498	{
499	int ret;
500
501	if (qiov->size == 0) {
502	return 0;
503	}
504
505	BLKDBG_CO_EVENT(bs->file, BLKDBG_COW_READ);
506
507	if (!bs->drv) {
508	return -ENOMEDIUM;
509	}
510
511	/*
512	* We never deal with requests that don't satisfy
513	* bdrv_check_qiov_request(), and aligning requests to clusters never
514	* breaks this condition. So, do some assertions before calling
515	* bs->drv->bdrv_co_preadv_part() which has int64_t arguments.
516	*/
517	assert(src_cluster_offset <= INT64_MAX);
518	assert(src_cluster_offset + offset_in_cluster <= INT64_MAX);
519	/* Cast qiov->size to uint64_t to silence a compiler warning on -m32 */
520	assert((uint64_t)qiov->size <= INT64_MAX);
521	bdrv_check_qiov_request(src_cluster_offset + offset_in_cluster, qiov->size,
522	qiov, 0, &error_abort);
523	/*
524	* Call .bdrv_co_readv() directly instead of using the public block-layer
525	* interface. This avoids double I/O throttling and request tracking,
526	* which can lead to deadlock when block layer copy-on-read is enabled.
527	*/
528	ret = bs->drv->bdrv_co_preadv_part(bs,
529	src_cluster_offset + offset_in_cluster,
530	qiov->size, qiov, 0, 0);
531	if (ret < 0) {
532	return ret;
533	}
534
535	return 0;
536	}
537
538	static int coroutine_fn GRAPH_RDLOCK
539	do_perform_cow_write(BlockDriverState *bs, uint64_t cluster_offset,
540	unsigned offset_in_cluster, QEMUIOVector *qiov)
541	{
542	BDRVQcow2State *s = bs->opaque;
543	int ret;
544
545	if (qiov->size == 0) {
546	return 0;
547	}
548
549	ret = qcow2_pre_write_overlap_check(bs, 0,
550	cluster_offset + offset_in_cluster, qiov->size, true);
551	if (ret < 0) {
552	return ret;
553	}
554
555	BLKDBG_CO_EVENT(bs->file, BLKDBG_COW_WRITE);
556	ret = bdrv_co_pwritev(s->data_file, cluster_offset + offset_in_cluster,
557	qiov->size, qiov, 0);
558	if (ret < 0) {
559	return ret;
560	}
561
562	return 0;
563	}
564
565
566	/*
567	* get_host_offset
568	*
569	* For a given offset of the virtual disk find the equivalent host
570	* offset in the qcow2 file and store it in *host_offset. Neither
571	* offset needs to be aligned to a cluster boundary.
572	*
573	* If the cluster is unallocated then *host_offset will be 0.
574	* If the cluster is compressed then *host_offset will contain the l2 entry.
575	*
576	* On entry, *bytes is the maximum number of contiguous bytes starting at
577	* offset that we are interested in.
578	*
579	* On exit, *bytes is the number of bytes starting at offset that have the same
580	* subcluster type and (if applicable) are stored contiguously in the image
581	* file. The subcluster type is stored in *subcluster_type.
582	* Compressed clusters are always processed one by one.
583	*
584	* Returns 0 on success, -errno in error cases.
585	*/
586	int qcow2_get_host_offset(BlockDriverState *bs, uint64_t offset,
587	unsigned int bytes, uint64_t host_offset,
588	QCow2SubclusterType *subcluster_type)
589	{
590	BDRVQcow2State *s = bs->opaque;
591	unsigned int l2_index, sc_index;
592	uint64_t l1_index, l2_offset, *l2_slice, l2_entry, l2_bitmap;
593	int sc;
594	unsigned int offset_in_cluster;
595	uint64_t bytes_available, bytes_needed, nb_clusters;
596	QCow2SubclusterType type;
597	int ret;
598
599	offset_in_cluster = offset_into_cluster(s, offset);
600	bytes_needed = (uint64_t) *bytes + offset_in_cluster;
601
602	/* compute how many bytes there are between the start of the cluster
603	* containing offset and the end of the l2 slice that contains
604	* the entry pointing to it */
605	bytes_available =
606	((uint64_t) (s->l2_slice_size - offset_to_l2_slice_index(s, offset)))
607	<< s->cluster_bits;
608
609	if (bytes_needed > bytes_available) {
610	bytes_needed = bytes_available;
611	}
612
613	*host_offset = 0;
614
615	/* seek to the l2 offset in the l1 table */
616
617	l1_index = offset_to_l1_index(s, offset);
618	if (l1_index >= s->l1_size) {
619	type = QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN;
620	goto out;
621	}
622
623	l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
624	if (!l2_offset) {
625	type = QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN;
626	goto out;
627	}
628
629	if (offset_into_cluster(s, l2_offset)) {
630	qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
631	" unaligned (L1 index: %#" PRIx64 ")",
632	l2_offset, l1_index);
633	return -EIO;
634	}
635
636	/* load the l2 slice in memory */
637
638	ret = l2_load(bs, offset, l2_offset, &l2_slice);
639	if (ret < 0) {
640	return ret;
641	}
642
643	/* find the cluster offset for the given disk offset */
644
645	l2_index = offset_to_l2_slice_index(s, offset);
646	sc_index = offset_to_sc_index(s, offset);
647	l2_entry = get_l2_entry(s, l2_slice, l2_index);
648	l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
649
650	nb_clusters = size_to_clusters(s, bytes_needed);
651	/* bytes_needed <= *bytes + offset_in_cluster, both of which are unsigned
652	* integers; the minimum cluster size is 512, so this assertion is always
653	* true */
654	assert(nb_clusters <= INT_MAX);
655
656	type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
657	if (s->qcow_version < 3 && (type == QCOW2_SUBCLUSTER_ZERO_PLAIN \|\|
658	type == QCOW2_SUBCLUSTER_ZERO_ALLOC)) {
659	qcow2_signal_corruption(bs, true, -1, -1, "Zero cluster entry found"
660	" in pre-v3 image (L2 offset: %#" PRIx64
661	", L2 index: %#x)", l2_offset, l2_index);
662	ret = -EIO;
663	goto fail;
664	}
665	switch (type) {
666	case QCOW2_SUBCLUSTER_INVALID:
667	break; /* This is handled by count_contiguous_subclusters() below */
668	case QCOW2_SUBCLUSTER_COMPRESSED:
669	if (has_data_file(bs)) {
670	qcow2_signal_corruption(bs, true, -1, -1, "Compressed cluster "
671	"entry found in image with external data "
672	"file (L2 offset: %#" PRIx64 ", L2 index: "
673	"%#x)", l2_offset, l2_index);
674	ret = -EIO;
675	goto fail;
676	}
677	*host_offset = l2_entry;
678	break;
679	case QCOW2_SUBCLUSTER_ZERO_PLAIN:
680	case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
681	break;
682	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
683	case QCOW2_SUBCLUSTER_NORMAL:
684	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC: {
685	uint64_t host_cluster_offset = l2_entry & L2E_OFFSET_MASK;
686	*host_offset = host_cluster_offset + offset_in_cluster;
687	if (offset_into_cluster(s, host_cluster_offset)) {
688	qcow2_signal_corruption(bs, true, -1, -1,
689	"Cluster allocation offset %#"
690	PRIx64 " unaligned (L2 offset: %#" PRIx64
691	", L2 index: %#x)", host_cluster_offset,
692	l2_offset, l2_index);
693	ret = -EIO;
694	goto fail;
695	}
696	if (has_data_file(bs) && *host_offset != offset) {
697	qcow2_signal_corruption(bs, true, -1, -1,
698	"External data file host cluster offset %#"
699	PRIx64 " does not match guest cluster "
700	"offset: %#" PRIx64
701	", L2 index: %#x)", host_cluster_offset,
702	offset - offset_in_cluster, l2_index);
703	ret = -EIO;
704	goto fail;
705	}
706	break;
707	}
708	default:
709	abort();
710	}
711
712	sc = count_contiguous_subclusters(bs, nb_clusters, sc_index,
713	l2_slice, &l2_index);
714	if (sc < 0) {
715	qcow2_signal_corruption(bs, true, -1, -1, "Invalid cluster entry found "
716	" (L2 offset: %#" PRIx64 ", L2 index: %#x)",
717	l2_offset, l2_index);
718	ret = -EIO;
719	goto fail;
720	}
721	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
722
723	bytes_available = ((int64_t)sc + sc_index) << s->subcluster_bits;
724
725	out:
726	if (bytes_available > bytes_needed) {
727	bytes_available = bytes_needed;
728	}
729
730	/* bytes_available <= bytes_needed <= *bytes + offset_in_cluster;
731	* subtracting offset_in_cluster will therefore definitely yield something
732	* not exceeding UINT_MAX */
733	assert(bytes_available - offset_in_cluster <= UINT_MAX);
734	*bytes = bytes_available - offset_in_cluster;
735
736	*subcluster_type = type;
737
738	return 0;
739
740	fail:
741	qcow2_cache_put(s->l2_table_cache, (void **)&l2_slice);
742	return ret;
743	}
744
745	/*
746	* get_cluster_table
747	*
748	* for a given disk offset, load (and allocate if needed)
749	* the appropriate slice of its l2 table.
750	*
751	* the cluster index in the l2 slice is given to the caller.
752	*
753	* Returns 0 on success, -errno in failure case
754	*/
755	static int GRAPH_RDLOCK
756	get_cluster_table(BlockDriverState *bs, uint64_t offset,
757	uint64_t *new_l2_slice, int new_l2_index)
758	{
759	BDRVQcow2State *s = bs->opaque;
760	unsigned int l2_index;
761	uint64_t l1_index, l2_offset;
762	uint64_t *l2_slice = NULL;
763	int ret;
764
765	/* seek to the l2 offset in the l1 table */
766
767	l1_index = offset_to_l1_index(s, offset);
768	if (l1_index >= s->l1_size) {
769	ret = qcow2_grow_l1_table(bs, l1_index + 1, false);
770	if (ret < 0) {
771	return ret;
772	}
773	}
774
775	assert(l1_index < s->l1_size);
776	l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
777	if (offset_into_cluster(s, l2_offset)) {
778	qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
779	" unaligned (L1 index: %#" PRIx64 ")",
780	l2_offset, l1_index);
781	return -EIO;
782	}
783
784	if (!(s->l1_table[l1_index] & QCOW_OFLAG_COPIED)) {
785	/* First allocate a new L2 table (and do COW if needed) */
786	ret = l2_allocate(bs, l1_index);
787	if (ret < 0) {
788	return ret;
789	}
790
791	/* Then decrease the refcount of the old table */
792	if (l2_offset) {
793	qcow2_free_clusters(bs, l2_offset, s->l2_size * l2_entry_size(s),
794	QCOW2_DISCARD_OTHER);
795	}
796
797	/* Get the offset of the newly-allocated l2 table */
798	l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
799	assert(offset_into_cluster(s, l2_offset) == 0);
800	}
801
802	/* load the l2 slice in memory */
803	ret = l2_load(bs, offset, l2_offset, &l2_slice);
804	if (ret < 0) {
805	return ret;
806	}
807
808	/* find the cluster offset for the given disk offset */
809
810	l2_index = offset_to_l2_slice_index(s, offset);
811
812	*new_l2_slice = l2_slice;
813	*new_l2_index = l2_index;
814
815	return 0;
816	}
817
818	/*
819	* alloc_compressed_cluster_offset
820	*
821	* For a given offset on the virtual disk, allocate a new compressed cluster
822	* and put the host offset of the cluster into *host_offset. If a cluster is
823	* already allocated at the offset, return an error.
824	*
825	* Return 0 on success and -errno in error cases
826	*/
827	int coroutine_fn GRAPH_RDLOCK
828	qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs, uint64_t offset,
829	int compressed_size, uint64_t *host_offset)
830	{
831	BDRVQcow2State *s = bs->opaque;
832	int l2_index, ret;
833	uint64_t *l2_slice;
834	int64_t cluster_offset;
835	int nb_csectors;
836
837	if (has_data_file(bs)) {
838	return 0;
839	}
840
841	ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
842	if (ret < 0) {
843	return ret;
844	}
845
846	/* Compression can't overwrite anything. Fail if the cluster was already
847	* allocated. */
848	cluster_offset = get_l2_entry(s, l2_slice, l2_index);
849	if (cluster_offset & L2E_OFFSET_MASK) {
850	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
851	return -EIO;
852	}
853
854	cluster_offset = qcow2_alloc_bytes(bs, compressed_size);
855	if (cluster_offset < 0) {
856	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
857	return cluster_offset;
858	}
859
860	nb_csectors =
861	(cluster_offset + compressed_size - 1) / QCOW2_COMPRESSED_SECTOR_SIZE -
862	(cluster_offset / QCOW2_COMPRESSED_SECTOR_SIZE);
863
864	/* The offset and size must fit in their fields of the L2 table entry */
865	assert((cluster_offset & s->cluster_offset_mask) == cluster_offset);
866	assert((nb_csectors & s->csize_mask) == nb_csectors);
867
868	cluster_offset \|= QCOW_OFLAG_COMPRESSED \|
869	((uint64_t)nb_csectors << s->csize_shift);
870
871	/* update L2 table */
872
873	/* compressed clusters never have the copied flag */
874
875	BLKDBG_CO_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);
876	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
877	set_l2_entry(s, l2_slice, l2_index, cluster_offset);
878	if (has_subclusters(s)) {
879	set_l2_bitmap(s, l2_slice, l2_index, 0);
880	}
881	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
882
883	*host_offset = cluster_offset & s->cluster_offset_mask;
884	return 0;
885	}
886
887	static int coroutine_fn GRAPH_RDLOCK
888	perform_cow(BlockDriverState bs, QCowL2Meta m)
889	{
890	BDRVQcow2State *s = bs->opaque;
891	Qcow2COWRegion *start = &m->cow_start;
892	Qcow2COWRegion *end = &m->cow_end;
893	unsigned buffer_size;
894	unsigned data_bytes = end->offset - (start->offset + start->nb_bytes);
895	bool merge_reads;
896	uint8_t start_buffer, end_buffer;
897	QEMUIOVector qiov;
898	int ret;
899
900	assert(start->nb_bytes <= UINT_MAX - end->nb_bytes);
901	assert(start->nb_bytes + end->nb_bytes <= UINT_MAX - data_bytes);
902	assert(start->offset + start->nb_bytes <= end->offset);
903
904	if ((start->nb_bytes == 0 && end->nb_bytes == 0) \|\| m->skip_cow) {
905	return 0;
906	}
907
908	/* If we have to read both the start and end COW regions and the
909	* middle region is not too large then perform just one read
910	* operation */
911	merge_reads = start->nb_bytes && end->nb_bytes && data_bytes <= 16384;
912	if (merge_reads) {
913	buffer_size = start->nb_bytes + data_bytes + end->nb_bytes;
914	} else {
915	/* If we have to do two reads, add some padding in the middle
916	* if necessary to make sure that the end region is optimally
917	* aligned. */
918	size_t align = bdrv_opt_mem_align(bs);
919	assert(align > 0 && align <= UINT_MAX);
920	assert(QEMU_ALIGN_UP(start->nb_bytes, align) <=
921	UINT_MAX - end->nb_bytes);
922	buffer_size = QEMU_ALIGN_UP(start->nb_bytes, align) + end->nb_bytes;
923	}
924
925	/* Reserve a buffer large enough to store all the data that we're
926	* going to read */
927	start_buffer = qemu_try_blockalign(bs, buffer_size);
928	if (start_buffer == NULL) {
929	return -ENOMEM;
930	}
931	/* The part of the buffer where the end region is located */
932	end_buffer = start_buffer + buffer_size - end->nb_bytes;
933
934	qemu_iovec_init(&qiov, 2 + (m->data_qiov ?
935	qemu_iovec_subvec_niov(m->data_qiov,
936	m->data_qiov_offset,
937	data_bytes)
938	: 0));
939
940	qemu_co_mutex_unlock(&s->lock);
941	/* First we read the existing data from both COW regions. We
942	* either read the whole region in one go, or the start and end
943	* regions separately. */
944	if (merge_reads) {
945	qemu_iovec_add(&qiov, start_buffer, buffer_size);
946	ret = do_perform_cow_read(bs, m->offset, start->offset, &qiov);
947	} else {
948	qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
949	ret = do_perform_cow_read(bs, m->offset, start->offset, &qiov);
950	if (ret < 0) {
951	goto fail;
952	}
953
954	qemu_iovec_reset(&qiov);
955	qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
956	ret = do_perform_cow_read(bs, m->offset, end->offset, &qiov);
957	}
958	if (ret < 0) {
959	goto fail;
960	}
961
962	/* Encrypt the data if necessary before writing it */
963	if (bs->encrypted) {
964	ret = qcow2_co_encrypt(bs,
965	m->alloc_offset + start->offset,
966	m->offset + start->offset,
967	start_buffer, start->nb_bytes);
968	if (ret < 0) {
969	goto fail;
970	}
971
972	ret = qcow2_co_encrypt(bs,
973	m->alloc_offset + end->offset,
974	m->offset + end->offset,
975	end_buffer, end->nb_bytes);
976	if (ret < 0) {
977	goto fail;
978	}
979	}
980
981	/* And now we can write everything. If we have the guest data we
982	* can write everything in one single operation */
983	if (m->data_qiov) {
984	qemu_iovec_reset(&qiov);
985	if (start->nb_bytes) {
986	qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
987	}
988	qemu_iovec_concat(&qiov, m->data_qiov, m->data_qiov_offset, data_bytes);
989	if (end->nb_bytes) {
990	qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
991	}
992	/* NOTE: we have a write_aio blkdebug event here followed by
993	* a cow_write one in do_perform_cow_write(), but there's only
994	* one single I/O operation */
995	BLKDBG_CO_EVENT(bs->file, BLKDBG_WRITE_AIO);
996	ret = do_perform_cow_write(bs, m->alloc_offset, start->offset, &qiov);
997	} else {
998	/* If there's no guest data then write both COW regions separately */
999	qemu_iovec_reset(&qiov);
1000	qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
1001	ret = do_perform_cow_write(bs, m->alloc_offset, start->offset, &qiov);
1002	if (ret < 0) {
1003	goto fail;
1004	}
1005
1006	qemu_iovec_reset(&qiov);
1007	qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
1008	ret = do_perform_cow_write(bs, m->alloc_offset, end->offset, &qiov);
1009	}
1010
1011	fail:
1012	qemu_co_mutex_lock(&s->lock);
1013
1014	/*
1015	* Before we update the L2 table to actually point to the new cluster, we
1016	* need to be sure that the refcounts have been increased and COW was
1017	* handled.
1018	*/
1019	if (ret == 0) {
1020	qcow2_cache_depends_on_flush(s->l2_table_cache);
1021	}
1022
1023	qemu_vfree(start_buffer);
1024	qemu_iovec_destroy(&qiov);
1025	return ret;
1026	}
1027
1028	int coroutine_fn qcow2_alloc_cluster_link_l2(BlockDriverState *bs,
1029	QCowL2Meta *m)
1030	{
1031	BDRVQcow2State *s = bs->opaque;
1032	int i, j = 0, l2_index, ret;
1033	uint64_t old_cluster, l2_slice;
1034	uint64_t cluster_offset = m->alloc_offset;
1035
1036	trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
1037	assert(m->nb_clusters > 0);
1038
1039	old_cluster = g_try_new(uint64_t, m->nb_clusters);
1040	if (old_cluster == NULL) {
1041	ret = -ENOMEM;
1042	goto err;
1043	}
1044
1045	/* copy content of unmodified sectors */
1046	ret = perform_cow(bs, m);
1047	if (ret < 0) {
1048	goto err;
1049	}
1050
1051	/* Update L2 table. */
1052	if (s->use_lazy_refcounts) {
1053	qcow2_mark_dirty(bs);
1054	}
1055	if (qcow2_need_accurate_refcounts(s)) {
1056	qcow2_cache_set_dependency(bs, s->l2_table_cache,
1057	s->refcount_block_cache);
1058	}
1059
1060	ret = get_cluster_table(bs, m->offset, &l2_slice, &l2_index);
1061	if (ret < 0) {
1062	goto err;
1063	}
1064	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
1065
1066	assert(l2_index + m->nb_clusters <= s->l2_slice_size);
1067	assert(m->cow_end.offset + m->cow_end.nb_bytes <=
1068	m->nb_clusters << s->cluster_bits);
1069	for (i = 0; i < m->nb_clusters; i++) {
1070	uint64_t offset = cluster_offset + ((uint64_t)i << s->cluster_bits);
1071	/* if two concurrent writes happen to the same unallocated cluster
1072	* each write allocates separate cluster and writes data concurrently.
1073	* The first one to complete updates l2 table with pointer to its
1074	* cluster the second one has to do RMW (which is done above by
1075	* perform_cow()), update l2 table with its cluster pointer and free
1076	* old cluster. This is what this loop does */
1077	if (get_l2_entry(s, l2_slice, l2_index + i) != 0) {
1078	old_cluster[j++] = get_l2_entry(s, l2_slice, l2_index + i);
1079	}
1080
1081	/* The offset must fit in the offset field of the L2 table entry */
1082	assert((offset & L2E_OFFSET_MASK) == offset);
1083
1084	set_l2_entry(s, l2_slice, l2_index + i, offset \| QCOW_OFLAG_COPIED);
1085
1086	/* Update bitmap with the subclusters that were just written */
1087	if (has_subclusters(s) && !m->prealloc) {
1088	uint64_t l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
1089	unsigned written_from = m->cow_start.offset;
1090	unsigned written_to = m->cow_end.offset + m->cow_end.nb_bytes;
1091	int first_sc, last_sc;
1092	/* Narrow written_from and written_to down to the current cluster */
1093	written_from = MAX(written_from, i << s->cluster_bits);
1094	written_to = MIN(written_to, (i + 1) << s->cluster_bits);
1095	assert(written_from < written_to);
1096	first_sc = offset_to_sc_index(s, written_from);
1097	last_sc = offset_to_sc_index(s, written_to - 1);
1098	l2_bitmap \|= QCOW_OFLAG_SUB_ALLOC_RANGE(first_sc, last_sc + 1);
1099	l2_bitmap &= ~QCOW_OFLAG_SUB_ZERO_RANGE(first_sc, last_sc + 1);
1100	set_l2_bitmap(s, l2_slice, l2_index + i, l2_bitmap);
1101	}
1102	}
1103
1104
1105	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1106
1107	/*
1108	* If this was a COW, we need to decrease the refcount of the old cluster.
1109	*
1110	* Don't discard clusters that reach a refcount of 0 (e.g. compressed
1111	* clusters), the next write will reuse them anyway.
1112	*/
1113	if (!m->keep_old_clusters && j != 0) {
1114	for (i = 0; i < j; i++) {
1115	qcow2_free_any_cluster(bs, old_cluster[i], QCOW2_DISCARD_NEVER);
1116	}
1117	}
1118
1119	ret = 0;
1120	err:
1121	g_free(old_cluster);
1122	return ret;
1123	}
1124
1125	/**
1126	* Frees the allocated clusters because the request failed and they won't
1127	* actually be linked.
1128	*/
1129	void coroutine_fn qcow2_alloc_cluster_abort(BlockDriverState bs, QCowL2Meta m)
1130	{
1131	BDRVQcow2State *s = bs->opaque;
1132	if (!has_data_file(bs) && !m->keep_old_clusters) {
1133	qcow2_free_clusters(bs, m->alloc_offset,
1134	m->nb_clusters << s->cluster_bits,
1135	QCOW2_DISCARD_NEVER);
1136	}
1137	}
1138
1139	/*
1140	* For a given write request, create a new QCowL2Meta structure, add
1141	* it to @m and the BDRVQcow2State.cluster_allocs list. If the write
1142	* request does not need copy-on-write or changes to the L2 metadata
1143	* then this function does nothing.
1144	*
1145	* @host_cluster_offset points to the beginning of the first cluster.
1146	*
1147	* @guest_offset and @bytes indicate the offset and length of the
1148	* request.
1149	*
1150	* @l2_slice contains the L2 entries of all clusters involved in this
1151	* write request.
1152	*
1153	* If @keep_old is true it means that the clusters were already
1154	* allocated and will be overwritten. If false then the clusters are
1155	* new and we have to decrease the reference count of the old ones.
1156	*
1157	* Returns 0 on success, -errno on failure.
1158	*/
1159	static int coroutine_fn GRAPH_RDLOCK
1160	calculate_l2_meta(BlockDriverState *bs, uint64_t host_cluster_offset,
1161	uint64_t guest_offset, unsigned bytes, uint64_t *l2_slice,
1162	QCowL2Meta **m, bool keep_old)
1163	{
1164	BDRVQcow2State *s = bs->opaque;
1165	int sc_index, l2_index = offset_to_l2_slice_index(s, guest_offset);
1166	uint64_t l2_entry, l2_bitmap;
1167	unsigned cow_start_from, cow_end_to;
1168	unsigned cow_start_to = offset_into_cluster(s, guest_offset);
1169	unsigned cow_end_from = cow_start_to + bytes;
1170	unsigned nb_clusters = size_to_clusters(s, cow_end_from);
1171	QCowL2Meta old_m = m;
1172	QCow2SubclusterType type;
1173	int i;
1174	bool skip_cow = keep_old;
1175
1176	assert(nb_clusters <= s->l2_slice_size - l2_index);
1177
1178	/* Check the type of all affected subclusters */
1179	for (i = 0; i < nb_clusters; i++) {
1180	l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1181	l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
1182	if (skip_cow) {
1183	unsigned write_from = MAX(cow_start_to, i << s->cluster_bits);
1184	unsigned write_to = MIN(cow_end_from, (i + 1) << s->cluster_bits);
1185	int first_sc = offset_to_sc_index(s, write_from);
1186	int last_sc = offset_to_sc_index(s, write_to - 1);
1187	int cnt = qcow2_get_subcluster_range_type(bs, l2_entry, l2_bitmap,
1188	first_sc, &type);
1189	/* Is any of the subclusters of type != QCOW2_SUBCLUSTER_NORMAL ? */
1190	if (type != QCOW2_SUBCLUSTER_NORMAL \|\| first_sc + cnt <= last_sc) {
1191	skip_cow = false;
1192	}
1193	} else {
1194	/* If we can't skip the cow we can still look for invalid entries */
1195	type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, 0);
1196	}
1197	if (type == QCOW2_SUBCLUSTER_INVALID) {
1198	int l1_index = offset_to_l1_index(s, guest_offset);
1199	uint64_t l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
1200	qcow2_signal_corruption(bs, true, -1, -1, "Invalid cluster "
1201	"entry found (L2 offset: %#" PRIx64
1202	", L2 index: %#x)",
1203	l2_offset, l2_index + i);
1204	return -EIO;
1205	}
1206	}
1207
1208	if (skip_cow) {
1209	return 0;
1210	}
1211
1212	/* Get the L2 entry of the first cluster */
1213	l2_entry = get_l2_entry(s, l2_slice, l2_index);
1214	l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
1215	sc_index = offset_to_sc_index(s, guest_offset);
1216	type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
1217
1218	if (!keep_old) {
1219	switch (type) {
1220	case QCOW2_SUBCLUSTER_COMPRESSED:
1221	cow_start_from = 0;
1222	break;
1223	case QCOW2_SUBCLUSTER_NORMAL:
1224	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1225	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1226	if (has_subclusters(s)) {
1227	/* Skip all leading zero and unallocated subclusters */
1228	uint32_t alloc_bitmap = l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC;
1229	cow_start_from =
1230	MIN(sc_index, ctz32(alloc_bitmap)) << s->subcluster_bits;
1231	} else {
1232	cow_start_from = 0;
1233	}
1234	break;
1235	case QCOW2_SUBCLUSTER_ZERO_PLAIN:
1236	case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
1237	cow_start_from = sc_index << s->subcluster_bits;
1238	break;
1239	default:
1240	g_assert_not_reached();
1241	}
1242	} else {
1243	switch (type) {
1244	case QCOW2_SUBCLUSTER_NORMAL:
1245	cow_start_from = cow_start_to;
1246	break;
1247	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1248	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1249	cow_start_from = sc_index << s->subcluster_bits;
1250	break;
1251	default:
1252	g_assert_not_reached();
1253	}
1254	}
1255
1256	/* Get the L2 entry of the last cluster */
1257	l2_index += nb_clusters - 1;
1258	l2_entry = get_l2_entry(s, l2_slice, l2_index);
1259	l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
1260	sc_index = offset_to_sc_index(s, guest_offset + bytes - 1);
1261	type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
1262
1263	if (!keep_old) {
1264	switch (type) {
1265	case QCOW2_SUBCLUSTER_COMPRESSED:
1266	cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);
1267	break;
1268	case QCOW2_SUBCLUSTER_NORMAL:
1269	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1270	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1271	cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);
1272	if (has_subclusters(s)) {
1273	/* Skip all trailing zero and unallocated subclusters */
1274	uint32_t alloc_bitmap = l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC;
1275	cow_end_to -=
1276	MIN(s->subclusters_per_cluster - sc_index - 1,
1277	clz32(alloc_bitmap)) << s->subcluster_bits;
1278	}
1279	break;
1280	case QCOW2_SUBCLUSTER_ZERO_PLAIN:
1281	case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
1282	cow_end_to = ROUND_UP(cow_end_from, s->subcluster_size);
1283	break;
1284	default:
1285	g_assert_not_reached();
1286	}
1287	} else {
1288	switch (type) {
1289	case QCOW2_SUBCLUSTER_NORMAL:
1290	cow_end_to = cow_end_from;
1291	break;
1292	case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1293	case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1294	cow_end_to = ROUND_UP(cow_end_from, s->subcluster_size);
1295	break;
1296	default:
1297	g_assert_not_reached();
1298	}
1299	}
1300
1301	m = g_malloc0(sizeof(*m));
1302	**m = (QCowL2Meta) {
1303	.next = old_m,
1304
1305	.alloc_offset = host_cluster_offset,
1306	.offset = start_of_cluster(s, guest_offset),
1307	.nb_clusters = nb_clusters,
1308
1309	.keep_old_clusters = keep_old,
1310
1311	.cow_start = {
1312	.offset = cow_start_from,
1313	.nb_bytes = cow_start_to - cow_start_from,
1314	},
1315	.cow_end = {
1316	.offset = cow_end_from,
1317	.nb_bytes = cow_end_to - cow_end_from,
1318	},
1319	};
1320
1321	qemu_co_queue_init(&(*m)->dependent_requests);
1322	QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
1323
1324	return 0;
1325	}
1326
1327	/*
1328	* Returns true if writing to the cluster pointed to by @l2_entry
1329	* requires a new allocation (that is, if the cluster is unallocated
1330	* or has refcount > 1 and therefore cannot be written in-place).
1331	*/
1332	static bool GRAPH_RDLOCK
1333	cluster_needs_new_alloc(BlockDriverState *bs, uint64_t l2_entry)
1334	{
1335	switch (qcow2_get_cluster_type(bs, l2_entry)) {
1336	case QCOW2_CLUSTER_NORMAL:
1337	case QCOW2_CLUSTER_ZERO_ALLOC:
1338	if (l2_entry & QCOW_OFLAG_COPIED) {
1339	return false;
1340	}
1341	/* fallthrough */
1342	case QCOW2_CLUSTER_UNALLOCATED:
1343	case QCOW2_CLUSTER_COMPRESSED:
1344	case QCOW2_CLUSTER_ZERO_PLAIN:
1345	return true;
1346	default:
1347	abort();
1348	}
1349	}
1350
1351	/*
1352	* Returns the number of contiguous clusters that can be written to
1353	* using one single write request, starting from @l2_index.
1354	* At most @nb_clusters are checked.
1355	*
1356	* If @new_alloc is true this counts clusters that are either
1357	* unallocated, or allocated but with refcount > 1 (so they need to be
1358	* newly allocated and COWed).
1359	*
1360	* If @new_alloc is false this counts clusters that are already
1361	* allocated and can be overwritten in-place (this includes clusters
1362	* of type QCOW2_CLUSTER_ZERO_ALLOC).
1363	*/
1364	static int GRAPH_RDLOCK
1365	count_single_write_clusters(BlockDriverState *bs, int nb_clusters,
1366	uint64_t *l2_slice, int l2_index, bool new_alloc)
1367	{
1368	BDRVQcow2State *s = bs->opaque;
1369	uint64_t l2_entry = get_l2_entry(s, l2_slice, l2_index);
1370	uint64_t expected_offset = l2_entry & L2E_OFFSET_MASK;
1371	int i;
1372
1373	for (i = 0; i < nb_clusters; i++) {
1374	l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1375	if (cluster_needs_new_alloc(bs, l2_entry) != new_alloc) {
1376	break;
1377	}
1378	if (!new_alloc) {
1379	if (expected_offset != (l2_entry & L2E_OFFSET_MASK)) {
1380	break;
1381	}
1382	expected_offset += s->cluster_size;
1383	}
1384	}
1385
1386	assert(i <= nb_clusters);
1387	return i;
1388	}
1389
1390	/*
1391	* Check if there already is an AIO write request in flight which allocates
1392	* the same cluster. In this case we need to wait until the previous
1393	* request has completed and updated the L2 table accordingly.
1394	*
1395	* If allow_shortening == true, instead of waiting for a dependency, *cur_bytes
1396	* can be shortened so that the cluster allocations don't overlap.
1397	*
1398	* Returns:
1399	* 0 if there was no dependency. *cur_bytes indicates the number of
1400	* bytes from guest_offset that can be read before the next
1401	* dependency must be processed (or the request is complete)
1402	*
1403	* -EAGAIN if we had to wait for another request, previously gathered
1404	* information on cluster allocation may be invalid now. The caller
1405	* must start over anyway, so consider *cur_bytes undefined.
1406	*/
1407	static int coroutine_fn handle_dependencies(BlockDriverState *bs,
1408	uint64_t guest_offset,
1409	uint64_t *cur_bytes,
1410	bool allow_shortening,
1411	QCowL2Meta **m)
1412	{
1413	BDRVQcow2State *s = bs->opaque;
1414	QCowL2Meta *old_alloc;
1415	uint64_t bytes = *cur_bytes;
1416
1417	QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
1418
1419	uint64_t start = guest_offset;
1420	uint64_t end = start + bytes;
1421	uint64_t old_start = start_of_cluster(s, l2meta_cow_start(old_alloc));
1422	uint64_t old_end = ROUND_UP(l2meta_cow_end(old_alloc), s->cluster_size);
1423
1424	if (end <= old_start \|\| start >= old_end) {
1425	/* No intersection */
1426	continue;
1427	}
1428
1429	if (old_alloc->keep_old_clusters &&
1430	(end <= l2meta_cow_start(old_alloc) \|\|
1431	start >= l2meta_cow_end(old_alloc)))
1432	{
1433	/*
1434	* Clusters intersect but COW areas don't. And cluster itself is
1435	* already allocated. So, there is no actual conflict.
1436	*/
1437	continue;
1438	}
1439
1440	/* Conflict */
1441
1442	if (start < old_start && allow_shortening) {
1443	/* Stop at the start of a running allocation */
1444	bytes = old_start - start;
1445	} else {
1446	bytes = 0;
1447	}
1448
1449	/*
1450	* Stop if an l2meta already exists. After yielding, it wouldn't
1451	* be valid any more, so we'd have to clean up the old L2Metas
1452	* and deal with requests depending on them before starting to
1453	* gather new ones. Not worth the trouble.
1454	*/
1455	if (bytes == 0 && *m) {
1456	*cur_bytes = 0;
1457	return 0;
1458	}
1459
1460	if (bytes == 0) {
1461	/*
1462	* Wait for the dependency to complete. We need to recheck
1463	* the free/allocated clusters when we continue.
1464	*/
1465	qemu_co_queue_wait(&old_alloc->dependent_requests, &s->lock);
1466	return -EAGAIN;
1467	}
1468	}
1469
1470	/* Make sure that existing clusters and new allocations are only used up to
1471	* the next dependency if we shortened the request above */
1472	*cur_bytes = bytes;
1473
1474	return 0;
1475	}
1476
1477	void coroutine_mixed_fn qcow2_wait_for_dependencies(BlockDriverState *bs,
1478	uint64_t guest_offset,
1479	uint64_t bytes)
1480	{
1481	BDRVQcow2State *s = bs->opaque;
1482	QCowL2Meta *m = NULL;
1483	int ret;
1484
1485	/*
1486	* Discard has some non-coroutine callers (creating internal snapshots and
1487	* make empty). They are calling from qemu-img or in a drained section, so
1488	* we know that no writes can be in progress.
1489	*/
1490	if (!qemu_in_coroutine()) {
1491	assert(QLIST_EMPTY(&s->cluster_allocs));
1492	return;
1493	}
1494
1495	do {
1496	ret = handle_dependencies(bs, guest_offset, &bytes, false, &m);
1497	} while (ret == -EAGAIN);
1498	}
1499
1500	/*
1501	* Checks how many already allocated clusters that don't require a new
1502	* allocation there are at the given guest_offset (up to *bytes).
1503	* If *host_offset is not INV_OFFSET, only physically contiguous clusters
1504	* beginning at this host offset are counted.
1505	*
1506	* Note that guest_offset may not be cluster aligned. In this case, the
1507	* returned *host_offset points to exact byte referenced by guest_offset and
1508	* therefore isn't cluster aligned as well.
1509	*
1510	* Returns:
1511	* 0: if no allocated clusters are available at the given offset.
1512	* *bytes is normally unchanged. It is set to 0 if the cluster
1513	* is allocated and can be overwritten in-place but doesn't have
1514	* the right physical offset.
1515	*
1516	* 1: if allocated clusters that can be overwritten in place are
1517	* available at the requested offset. *bytes may have decreased
1518	* and describes the length of the area that can be written to.
1519	*
1520	* -errno: in error cases
1521	*/
1522	static int coroutine_fn GRAPH_RDLOCK
1523	handle_copied(BlockDriverState *bs, uint64_t guest_offset,
1524	uint64_t host_offset, uint64_t bytes, QCowL2Meta **m)
1525	{
1526	BDRVQcow2State *s = bs->opaque;
1527	int l2_index;
1528	uint64_t l2_entry, cluster_offset;
1529	uint64_t *l2_slice;
1530	uint64_t nb_clusters;
1531	unsigned int keep_clusters;
1532	int ret;
1533
1534	trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,
1535	*bytes);
1536
1537	assert(*host_offset == INV_OFFSET \|\| offset_into_cluster(s, guest_offset)
1538	== offset_into_cluster(s, *host_offset));
1539
1540	/*
1541	* Calculate the number of clusters to look for. We stop at L2 slice
1542	* boundaries to keep things simple.
1543	*/
1544	nb_clusters =
1545	size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
1546
1547	l2_index = offset_to_l2_slice_index(s, guest_offset);
1548	nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1549	/* Limit total byte count to BDRV_REQUEST_MAX_BYTES */
1550	nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);
1551
1552	/* Find L2 entry for the first involved cluster */
1553	ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);
1554	if (ret < 0) {
1555	return ret;
1556	}
1557
1558	l2_entry = get_l2_entry(s, l2_slice, l2_index);
1559	cluster_offset = l2_entry & L2E_OFFSET_MASK;
1560
1561	if (!cluster_needs_new_alloc(bs, l2_entry)) {
1562	if (offset_into_cluster(s, cluster_offset)) {
1563	qcow2_signal_corruption(bs, true, -1, -1, "%s cluster offset "
1564	"%#" PRIx64 " unaligned (guest offset: %#"
1565	PRIx64 ")", l2_entry & QCOW_OFLAG_ZERO ?
1566	"Preallocated zero" : "Data",
1567	cluster_offset, guest_offset);
1568	ret = -EIO;
1569	goto out;
1570	}
1571
1572	/* If a specific host_offset is required, check it */
1573	if (host_offset != INV_OFFSET && cluster_offset != host_offset) {
1574	*bytes = 0;
1575	ret = 0;
1576	goto out;
1577	}
1578
1579	/* We keep all QCOW_OFLAG_COPIED clusters */
1580	keep_clusters = count_single_write_clusters(bs, nb_clusters, l2_slice,
1581	l2_index, false);
1582	assert(keep_clusters <= nb_clusters);
1583
1584	bytes = MIN(bytes,
1585	keep_clusters * s->cluster_size
1586	- offset_into_cluster(s, guest_offset));
1587	assert(*bytes != 0);
1588
1589	ret = calculate_l2_meta(bs, cluster_offset, guest_offset,
1590	*bytes, l2_slice, m, true);
1591	if (ret < 0) {
1592	goto out;
1593	}
1594
1595	ret = 1;
1596	} else {
1597	ret = 0;
1598	}
1599
1600	/* Cleanup */
1601	out:
1602	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1603
1604	/* Only return a host offset if we actually made progress. Otherwise we
1605	* would make requirements for handle_alloc() that it can't fulfill */
1606	if (ret > 0) {
1607	*host_offset = cluster_offset + offset_into_cluster(s, guest_offset);
1608	}
1609
1610	return ret;
1611	}
1612
1613	/*
1614	* Allocates new clusters for the given guest_offset.
1615	*
1616	* At most nb_clusters are allocated, and on return nb_clusters is updated to
1617	* contain the number of clusters that have been allocated and are contiguous
1618	* in the image file.
1619	*
1620	* If *host_offset is not INV_OFFSET, it specifies the offset in the image file
1621	* at which the new clusters must start. *nb_clusters can be 0 on return in
1622	* this case if the cluster at host_offset is already in use. If *host_offset
1623	* is INV_OFFSET, the clusters can be allocated anywhere in the image file.
1624	*
1625	* *host_offset is updated to contain the offset into the image file at which
1626	* the first allocated cluster starts.
1627	*
1628	* Return 0 on success and -errno in error cases. -EAGAIN means that the
1629	* function has been waiting for another request and the allocation must be
1630	* restarted, but the whole request should not be failed.
1631	*/
1632	static int coroutine_fn GRAPH_RDLOCK
1633	do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
1634	uint64_t host_offset, uint64_t nb_clusters)
1635	{
1636	BDRVQcow2State *s = bs->opaque;
1637
1638	trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
1639	host_offset, nb_clusters);
1640
1641	if (has_data_file(bs)) {
1642	assert(*host_offset == INV_OFFSET \|\|
1643	*host_offset == start_of_cluster(s, guest_offset));
1644	*host_offset = start_of_cluster(s, guest_offset);
1645	return 0;
1646	}
1647
1648	/* Allocate new clusters */
1649	trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
1650	if (*host_offset == INV_OFFSET) {
1651	int64_t cluster_offset =
1652	qcow2_alloc_clusters(bs, nb_clusters s->cluster_size);
1653	if (cluster_offset < 0) {
1654	return cluster_offset;
1655	}
1656	*host_offset = cluster_offset;
1657	return 0;
1658	} else {
1659	int64_t ret = qcow2_alloc_clusters_at(bs, host_offset, nb_clusters);
1660	if (ret < 0) {
1661	return ret;
1662	}
1663	*nb_clusters = ret;
1664	return 0;
1665	}
1666	}
1667
1668	/*
1669	* Allocates new clusters for an area that is either still unallocated or
1670	* cannot be overwritten in-place. If *host_offset is not INV_OFFSET,
1671	* clusters are only allocated if the new allocation can match the specified
1672	* host offset.
1673	*
1674	* Note that guest_offset may not be cluster aligned. In this case, the
1675	* returned *host_offset points to exact byte referenced by guest_offset and
1676	* therefore isn't cluster aligned as well.
1677	*
1678	* Returns:
1679	* 0: if no clusters could be allocated. *bytes is set to 0,
1680	* *host_offset is left unchanged.
1681	*
1682	* 1: if new clusters were allocated. *bytes may be decreased if the
1683	* new allocation doesn't cover all of the requested area.
1684	* *host_offset is updated to contain the host offset of the first
1685	* newly allocated cluster.
1686	*
1687	* -errno: in error cases
1688	*/
1689	static int coroutine_fn GRAPH_RDLOCK
1690	handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
1691	uint64_t host_offset, uint64_t bytes, QCowL2Meta **m)
1692	{
1693	BDRVQcow2State *s = bs->opaque;
1694	int l2_index;
1695	uint64_t *l2_slice;
1696	uint64_t nb_clusters;
1697	int ret;
1698
1699	uint64_t alloc_cluster_offset;
1700
1701	trace_qcow2_handle_alloc(qemu_coroutine_self(), guest_offset, *host_offset,
1702	*bytes);
1703	assert(*bytes > 0);
1704
1705	/*
1706	* Calculate the number of clusters to look for. We stop at L2 slice
1707	* boundaries to keep things simple.
1708	*/
1709	nb_clusters =
1710	size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
1711
1712	l2_index = offset_to_l2_slice_index(s, guest_offset);
1713	nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1714	/* Limit total allocation byte count to BDRV_REQUEST_MAX_BYTES */
1715	nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);
1716
1717	/* Find L2 entry for the first involved cluster */
1718	ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);
1719	if (ret < 0) {
1720	return ret;
1721	}
1722
1723	nb_clusters = count_single_write_clusters(bs, nb_clusters,
1724	l2_slice, l2_index, true);
1725
1726	/* This function is only called when there were no non-COW clusters, so if
1727	* we can't find any unallocated or COW clusters either, something is
1728	* wrong with our code. */
1729	assert(nb_clusters > 0);
1730
1731	/* Allocate at a given offset in the image file */
1732	alloc_cluster_offset = *host_offset == INV_OFFSET ? INV_OFFSET :
1733	start_of_cluster(s, *host_offset);
1734	ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,
1735	&nb_clusters);
1736	if (ret < 0) {
1737	goto out;
1738	}
1739
1740	/* Can't extend contiguous allocation */
1741	if (nb_clusters == 0) {
1742	*bytes = 0;
1743	ret = 0;
1744	goto out;
1745	}
1746
1747	assert(alloc_cluster_offset != INV_OFFSET);
1748
1749	/*
1750	* Save info needed for meta data update.
1751	*
1752	* requested_bytes: Number of bytes from the start of the first
1753	* newly allocated cluster to the end of the (possibly shortened
1754	* before) write request.
1755	*
1756	* avail_bytes: Number of bytes from the start of the first
1757	* newly allocated to the end of the last newly allocated cluster.
1758	*
1759	* nb_bytes: The number of bytes from the start of the first
1760	* newly allocated cluster to the end of the area that the write
1761	* request actually writes to (excluding COW at the end)
1762	*/
1763	uint64_t requested_bytes = *bytes + offset_into_cluster(s, guest_offset);
1764	int avail_bytes = nb_clusters << s->cluster_bits;
1765	int nb_bytes = MIN(requested_bytes, avail_bytes);
1766
1767	*host_offset = alloc_cluster_offset + offset_into_cluster(s, guest_offset);
1768	bytes = MIN(bytes, nb_bytes - offset_into_cluster(s, guest_offset));
1769	assert(*bytes != 0);
1770
1771	ret = calculate_l2_meta(bs, alloc_cluster_offset, guest_offset, *bytes,
1772	l2_slice, m, false);
1773	if (ret < 0) {
1774	goto out;
1775	}
1776
1777	ret = 1;
1778
1779	out:
1780	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1781	return ret;
1782	}
1783
1784	/*
1785	* For a given area on the virtual disk defined by @offset and @bytes,
1786	* find the corresponding area on the qcow2 image, allocating new
1787	* clusters (or subclusters) if necessary. The result can span a
1788	* combination of allocated and previously unallocated clusters.
1789	*
1790	* Note that offset may not be cluster aligned. In this case, the returned
1791	* *host_offset points to exact byte referenced by offset and therefore
1792	* isn't cluster aligned as well.
1793	*
1794	* On return, @host_offset is set to the beginning of the requested
1795	* area. This area is guaranteed to be contiguous on the qcow2 file
1796	* but it can be smaller than initially requested. In this case @bytes
1797	* is updated with the actual size.
1798	*
1799	* If any clusters or subclusters were allocated then @m contains a
1800	* list with the information of all the affected regions. Note that
1801	* this can happen regardless of whether this function succeeds or
1802	* not. The caller is responsible for updating the L2 metadata of the
1803	* allocated clusters (on success) or freeing them (on failure), and
1804	* for clearing the contents of @m afterwards in both cases.
1805	*
1806	* If the request conflicts with another write request in flight, the coroutine
1807	* is queued and will be reentered when the dependency has completed.
1808	*
1809	* Return 0 on success and -errno in error cases
1810	*/
1811	int coroutine_fn qcow2_alloc_host_offset(BlockDriverState *bs, uint64_t offset,
1812	unsigned int *bytes,
1813	uint64_t *host_offset,
1814	QCowL2Meta **m)
1815	{
1816	BDRVQcow2State *s = bs->opaque;
1817	uint64_t start, remaining;
1818	uint64_t cluster_offset;
1819	uint64_t cur_bytes;
1820	int ret;
1821
1822	trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset, *bytes);
1823
1824	again:
1825	start = offset;
1826	remaining = *bytes;
1827	cluster_offset = INV_OFFSET;
1828	*host_offset = INV_OFFSET;
1829	cur_bytes = 0;
1830	*m = NULL;
1831
1832	while (true) {
1833
1834	if (*host_offset == INV_OFFSET && cluster_offset != INV_OFFSET) {
1835	*host_offset = cluster_offset;
1836	}
1837
1838	assert(remaining >= cur_bytes);
1839
1840	start += cur_bytes;
1841	remaining -= cur_bytes;
1842
1843	if (cluster_offset != INV_OFFSET) {
1844	cluster_offset += cur_bytes;
1845	}
1846
1847	if (remaining == 0) {
1848	break;
1849	}
1850
1851	cur_bytes = remaining;
1852
1853	/*
1854	* Now start gathering as many contiguous clusters as possible:
1855	*
1856	* 1. Check for overlaps with in-flight allocations
1857	*
1858	* a) Overlap not in the first cluster -> shorten this request and
1859	* let the caller handle the rest in its next loop iteration.
1860	*
1861	* b) Real overlaps of two requests. Yield and restart the search
1862	* for contiguous clusters (the situation could have changed
1863	* while we were sleeping)
1864	*
1865	* c) TODO: Request starts in the same cluster as the in-flight
1866	* allocation ends. Shorten the COW of the in-fight allocation,
1867	* set cluster_offset to write to the same cluster and set up
1868	* the right synchronisation between the in-flight request and
1869	* the new one.
1870	*/
1871	ret = handle_dependencies(bs, start, &cur_bytes, true, m);
1872	if (ret == -EAGAIN) {
1873	/* Currently handle_dependencies() doesn't yield if we already had
1874	* an allocation. If it did, we would have to clean up the L2Meta
1875	* structs before starting over. */
1876	assert(*m == NULL);
1877	goto again;
1878	} else if (ret < 0) {
1879	return ret;
1880	} else if (cur_bytes == 0) {
1881	break;
1882	} else {
1883	/* handle_dependencies() may have decreased cur_bytes (shortened
1884	* the allocations below) so that the next dependency is processed
1885	* correctly during the next loop iteration. */
1886	}
1887
1888	/*
1889	* 2. Count contiguous COPIED clusters.
1890	*/
1891	ret = handle_copied(bs, start, &cluster_offset, &cur_bytes, m);
1892	if (ret < 0) {
1893	return ret;
1894	} else if (ret) {
1895	continue;
1896	} else if (cur_bytes == 0) {
1897	break;
1898	}
1899
1900	/*
1901	* 3. If the request still hasn't completed, allocate new clusters,
1902	* considering any cluster_offset of steps 1c or 2.
1903	*/
1904	ret = handle_alloc(bs, start, &cluster_offset, &cur_bytes, m);
1905	if (ret < 0) {
1906	return ret;
1907	} else if (ret) {
1908	continue;
1909	} else {
1910	assert(cur_bytes == 0);
1911	break;
1912	}
1913	}
1914
1915	*bytes -= remaining;
1916	assert(*bytes > 0);
1917	assert(*host_offset != INV_OFFSET);
1918	assert(offset_into_cluster(s, *host_offset) ==
1919	offset_into_cluster(s, offset));
1920
1921	return 0;
1922	}
1923
1924	/*
1925	* This discards as many clusters of nb_clusters as possible at once (i.e.
1926	* all clusters in the same L2 slice) and returns the number of discarded
1927	* clusters.
1928	*/
1929	static int GRAPH_RDLOCK
1930	discard_in_l2_slice(BlockDriverState *bs, uint64_t offset, uint64_t nb_clusters,
1931	enum qcow2_discard_type type, bool full_discard)
1932	{
1933	BDRVQcow2State *s = bs->opaque;
1934	uint64_t *l2_slice;
1935	int l2_index;
1936	int ret;
1937	int i;
1938
1939	ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
1940	if (ret < 0) {
1941	return ret;
1942	}
1943
1944	/* Limit nb_clusters to one L2 slice */
1945	nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1946	assert(nb_clusters <= INT_MAX);
1947
1948	for (i = 0; i < nb_clusters; i++) {
1949	uint64_t old_l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1950	uint64_t old_l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
1951	uint64_t new_l2_entry = old_l2_entry;
1952	uint64_t new_l2_bitmap = old_l2_bitmap;
1953	QCow2ClusterType cluster_type =
1954	qcow2_get_cluster_type(bs, old_l2_entry);
1955	bool keep_reference = (cluster_type != QCOW2_CLUSTER_COMPRESSED) &&
1956	!full_discard &&
1957	(s->discard_no_unref &&
1958	type == QCOW2_DISCARD_REQUEST);
1959
1960	/*
1961	* If full_discard is true, the cluster should not read back as zeroes,
1962	* but rather fall through to the backing file.
1963	*
1964	* If full_discard is false, make sure that a discarded area reads back
1965	* as zeroes for v3 images (we cannot do it for v2 without actually
1966	* writing a zero-filled buffer). We can skip the operation if the
1967	* cluster is already marked as zero, or if it's unallocated and we
1968	* don't have a backing file.
1969	*
1970	* TODO We might want to use bdrv_block_status(bs) here, but we're
1971	* holding s->lock, so that doesn't work today.
1972	*/
1973	if (full_discard) {
1974	new_l2_entry = new_l2_bitmap = 0;
1975	} else if (bs->backing \|\| qcow2_cluster_is_allocated(cluster_type)) {
1976	if (has_subclusters(s)) {
1977	if (keep_reference) {
1978	new_l2_entry = old_l2_entry;
1979	} else {
1980	new_l2_entry = 0;
1981	}
1982	new_l2_bitmap = QCOW_L2_BITMAP_ALL_ZEROES;
1983	} else {
1984	if (s->qcow_version >= 3) {
1985	if (keep_reference) {
1986	new_l2_entry \|= QCOW_OFLAG_ZERO;
1987	} else {
1988	new_l2_entry = QCOW_OFLAG_ZERO;
1989	}
1990	} else {
1991	new_l2_entry = 0;
1992	}
1993	}
1994	}
1995
1996	if (old_l2_entry == new_l2_entry && old_l2_bitmap == new_l2_bitmap) {
1997	continue;
1998	}
1999
2000	/* First remove L2 entries */
2001	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
2002	set_l2_entry(s, l2_slice, l2_index + i, new_l2_entry);
2003	if (has_subclusters(s)) {
2004	set_l2_bitmap(s, l2_slice, l2_index + i, new_l2_bitmap);
2005	}
2006	if (!keep_reference) {
2007	/* Then decrease the refcount */
2008	qcow2_free_any_cluster(bs, old_l2_entry, type);
2009	} else {
2010	/* If we keep the reference, pass on the discard still */
2011	qcow2_discard_cluster(bs, old_l2_entry & L2E_OFFSET_MASK,
2012	s->cluster_size, cluster_type, type);
2013	}
2014	}
2015
2016	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2017
2018	return nb_clusters;
2019	}
2020
2021	int qcow2_cluster_discard(BlockDriverState *bs, uint64_t offset,
2022	uint64_t bytes, enum qcow2_discard_type type,
2023	bool full_discard)
2024	{
2025	BDRVQcow2State *s = bs->opaque;
2026	uint64_t end_offset = offset + bytes;
2027	uint64_t nb_clusters;
2028	int64_t cleared;
2029	int ret;
2030
2031	/*
2032	* If we're touching a cluster for which allocating writes are in flight,
2033	* wait for them to complete to avoid conflicting metadata updates.
2034	*
2035	* We don't need to allocate a QCowL2Meta for the discard operation because
2036	* s->lock is held for the duration of the whole operation.
2037	*/
2038	qcow2_wait_for_dependencies(bs, offset, bytes);
2039
2040	/* Caller must pass aligned values, except at image end */
2041	assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
2042	assert(QEMU_IS_ALIGNED(end_offset, s->cluster_size) \|\|
2043	end_offset == bs->total_sectors << BDRV_SECTOR_BITS);
2044
2045	nb_clusters = size_to_clusters(s, bytes);
2046
2047	s->cache_discards = true;
2048
2049	/* Each L2 slice is handled by its own loop iteration */
2050	while (nb_clusters > 0) {
2051	cleared = discard_in_l2_slice(bs, offset, nb_clusters, type,
2052	full_discard);
2053	if (cleared < 0) {
2054	ret = cleared;
2055	goto fail;
2056	}
2057
2058	nb_clusters -= cleared;
2059	offset += (cleared * s->cluster_size);
2060	}
2061
2062	ret = 0;
2063	fail:
2064	s->cache_discards = false;
2065	qcow2_process_discards(bs, ret);
2066
2067	return ret;
2068	}
2069
2070	/*
2071	* This zeroes as many clusters of nb_clusters as possible at once (i.e.
2072	* all clusters in the same L2 slice) and returns the number of zeroed
2073	* clusters.
2074	*/
2075	static int coroutine_fn GRAPH_RDLOCK
2076	zero_in_l2_slice(BlockDriverState *bs, uint64_t offset,
2077	uint64_t nb_clusters, int flags)
2078	{
2079	BDRVQcow2State *s = bs->opaque;
2080	uint64_t *l2_slice;
2081	int l2_index;
2082	int ret;
2083	int i;
2084
2085	ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
2086	if (ret < 0) {
2087	return ret;
2088	}
2089
2090	/* Limit nb_clusters to one L2 slice */
2091	nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
2092	assert(nb_clusters <= INT_MAX);
2093
2094	for (i = 0; i < nb_clusters; i++) {
2095	uint64_t old_l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
2096	uint64_t old_l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
2097	QCow2ClusterType type = qcow2_get_cluster_type(bs, old_l2_entry);
2098	bool unmap = (type == QCOW2_CLUSTER_COMPRESSED) \|\|
2099	((flags & BDRV_REQ_MAY_UNMAP) && qcow2_cluster_is_allocated(type));
2100	bool keep_reference =
2101	(s->discard_no_unref && type != QCOW2_CLUSTER_COMPRESSED);
2102	uint64_t new_l2_entry = old_l2_entry;
2103	uint64_t new_l2_bitmap = old_l2_bitmap;
2104
2105	if (unmap && !keep_reference) {
2106	new_l2_entry = 0;
2107	}
2108
2109	if (has_subclusters(s)) {
2110	new_l2_bitmap = QCOW_L2_BITMAP_ALL_ZEROES;
2111	} else {
2112	new_l2_entry \|= QCOW_OFLAG_ZERO;
2113	}
2114
2115	if (old_l2_entry == new_l2_entry && old_l2_bitmap == new_l2_bitmap) {
2116	continue;
2117	}
2118
2119	/* First update L2 entries */
2120	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
2121	set_l2_entry(s, l2_slice, l2_index + i, new_l2_entry);
2122	if (has_subclusters(s)) {
2123	set_l2_bitmap(s, l2_slice, l2_index + i, new_l2_bitmap);
2124	}
2125
2126	if (unmap) {
2127	if (!keep_reference) {
2128	/* Then decrease the refcount */
2129	qcow2_free_any_cluster(bs, old_l2_entry, QCOW2_DISCARD_REQUEST);
2130	} else {
2131	/* If we keep the reference, pass on the discard still */
2132	qcow2_discard_cluster(bs, old_l2_entry & L2E_OFFSET_MASK,
2133	s->cluster_size, type, QCOW2_DISCARD_REQUEST);
2134	}
2135	}
2136	}
2137
2138	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2139
2140	return nb_clusters;
2141	}
2142
2143	static int coroutine_fn GRAPH_RDLOCK
2144	zero_l2_subclusters(BlockDriverState *bs, uint64_t offset,
2145	unsigned nb_subclusters)
2146	{
2147	BDRVQcow2State *s = bs->opaque;
2148	uint64_t *l2_slice;
2149	uint64_t old_l2_bitmap, l2_bitmap;
2150	int l2_index, ret, sc = offset_to_sc_index(s, offset);
2151
2152	/* For full clusters use zero_in_l2_slice() instead */
2153	assert(nb_subclusters > 0 && nb_subclusters < s->subclusters_per_cluster);
2154	assert(sc + nb_subclusters <= s->subclusters_per_cluster);
2155	assert(offset_into_subcluster(s, offset) == 0);
2156
2157	ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
2158	if (ret < 0) {
2159	return ret;
2160	}
2161
2162	switch (qcow2_get_cluster_type(bs, get_l2_entry(s, l2_slice, l2_index))) {
2163	case QCOW2_CLUSTER_COMPRESSED:
2164	ret = -ENOTSUP; /* We cannot partially zeroize compressed clusters */
2165	goto out;
2166	case QCOW2_CLUSTER_NORMAL:
2167	case QCOW2_CLUSTER_UNALLOCATED:
2168	break;
2169	default:
2170	g_assert_not_reached();
2171	}
2172
2173	old_l2_bitmap = l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
2174
2175	l2_bitmap \|= QCOW_OFLAG_SUB_ZERO_RANGE(sc, sc + nb_subclusters);
2176	l2_bitmap &= ~QCOW_OFLAG_SUB_ALLOC_RANGE(sc, sc + nb_subclusters);
2177
2178	if (old_l2_bitmap != l2_bitmap) {
2179	set_l2_bitmap(s, l2_slice, l2_index, l2_bitmap);
2180	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
2181	}
2182
2183	ret = 0;
2184	out:
2185	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2186
2187	return ret;
2188	}
2189
2190	int coroutine_fn qcow2_subcluster_zeroize(BlockDriverState *bs, uint64_t offset,
2191	uint64_t bytes, int flags)
2192	{
2193	BDRVQcow2State *s = bs->opaque;
2194	uint64_t end_offset = offset + bytes;
2195	uint64_t nb_clusters;
2196	unsigned head, tail;
2197	int64_t cleared;
2198	int ret;
2199
2200	/*
2201	* If we're touching a cluster for which allocating writes are in flight,
2202	* wait for them to complete to avoid conflicting metadata updates.
2203	*
2204	* We don't need to allocate a QCowL2Meta for the zeroize operation because
2205	* s->lock is held for the duration of the whole operation.
2206	*/
2207	qcow2_wait_for_dependencies(bs, offset, bytes);
2208
2209	/* If we have to stay in sync with an external data file, zero out
2210	* s->data_file first. */
2211	if (data_file_is_raw(bs)) {
2212	assert(has_data_file(bs));
2213	ret = bdrv_co_pwrite_zeroes(s->data_file, offset, bytes, flags);
2214	if (ret < 0) {
2215	return ret;
2216	}
2217	}
2218
2219	/* Caller must pass aligned values, except at image end */
2220	assert(offset_into_subcluster(s, offset) == 0);
2221	assert(offset_into_subcluster(s, end_offset) == 0 \|\|
2222	end_offset >= bs->total_sectors << BDRV_SECTOR_BITS);
2223
2224	/*
2225	* The zero flag is only supported by version 3 and newer. However, if we
2226	* have no backing file, we can resort to discard in version 2.
2227	*/
2228	if (s->qcow_version < 3) {
2229	if (!bs->backing) {
2230	return qcow2_cluster_discard(bs, offset, bytes,
2231	QCOW2_DISCARD_REQUEST, false);
2232	}
2233	return -ENOTSUP;
2234	}
2235
2236	head = MIN(end_offset, ROUND_UP(offset, s->cluster_size)) - offset;
2237	offset += head;
2238
2239	tail = (end_offset >= bs->total_sectors << BDRV_SECTOR_BITS) ? 0 :
2240	end_offset - MAX(offset, start_of_cluster(s, end_offset));
2241	end_offset -= tail;
2242
2243	s->cache_discards = true;
2244
2245	if (head) {
2246	ret = zero_l2_subclusters(bs, offset - head,
2247	size_to_subclusters(s, head));
2248	if (ret < 0) {
2249	goto fail;
2250	}
2251	}
2252
2253	/* Each L2 slice is handled by its own loop iteration */
2254	nb_clusters = size_to_clusters(s, end_offset - offset);
2255
2256	while (nb_clusters > 0) {
2257	cleared = zero_in_l2_slice(bs, offset, nb_clusters, flags);
2258	if (cleared < 0) {
2259	ret = cleared;
2260	goto fail;
2261	}
2262
2263	nb_clusters -= cleared;
2264	offset += (cleared * s->cluster_size);
2265	}
2266
2267	if (tail) {
2268	ret = zero_l2_subclusters(bs, end_offset, size_to_subclusters(s, tail));
2269	if (ret < 0) {
2270	goto fail;
2271	}
2272	}
2273
2274	ret = 0;
2275	fail:
2276	s->cache_discards = false;
2277	qcow2_process_discards(bs, ret);
2278
2279	return ret;
2280	}
2281
2282	/*
2283	* Expands all zero clusters in a specific L1 table (or deallocates them, for
2284	* non-backed non-pre-allocated zero clusters).
2285	*
2286	* l1_entries and *visited_l1_entries are used to keep track of progress for
2287	* status_cb(). l1_entries contains the total number of L1 entries and
2288	* *visited_l1_entries counts all visited L1 entries.
2289	*/
2290	static int GRAPH_RDLOCK
2291	expand_zero_clusters_in_l1(BlockDriverState bs, uint64_t l1_table,
2292	int l1_size, int64_t *visited_l1_entries,
2293	int64_t l1_entries,
2294	BlockDriverAmendStatusCB *status_cb,
2295	void *cb_opaque)
2296	{
2297	BDRVQcow2State *s = bs->opaque;
2298	bool is_active_l1 = (l1_table == s->l1_table);
2299	uint64_t *l2_slice = NULL;
2300	unsigned slice, slice_size2, n_slices;
2301	int ret;
2302	int i, j;
2303
2304	/* qcow2_downgrade() is not allowed in images with subclusters */
2305	assert(!has_subclusters(s));
2306
2307	slice_size2 = s->l2_slice_size * l2_entry_size(s);
2308	n_slices = s->cluster_size / slice_size2;
2309
2310	if (!is_active_l1) {
2311	/* inactive L2 tables require a buffer to be stored in when loading
2312	* them from disk */
2313	l2_slice = qemu_try_blockalign(bs->file->bs, slice_size2);
2314	if (l2_slice == NULL) {
2315	return -ENOMEM;
2316	}
2317	}
2318
2319	for (i = 0; i < l1_size; i++) {
2320	uint64_t l2_offset = l1_table[i] & L1E_OFFSET_MASK;
2321	uint64_t l2_refcount;
2322
2323	if (!l2_offset) {
2324	/* unallocated */
2325	(*visited_l1_entries)++;
2326	if (status_cb) {
2327	status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
2328	}
2329	continue;
2330	}
2331
2332	if (offset_into_cluster(s, l2_offset)) {
2333	qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#"
2334	PRIx64 " unaligned (L1 index: %#x)",
2335	l2_offset, i);
2336	ret = -EIO;
2337	goto fail;
2338	}
2339
2340	ret = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits,
2341	&l2_refcount);
2342	if (ret < 0) {
2343	goto fail;
2344	}
2345
2346	for (slice = 0; slice < n_slices; slice++) {
2347	uint64_t slice_offset = l2_offset + slice * slice_size2;
2348	bool l2_dirty = false;
2349	if (is_active_l1) {
2350	/* get active L2 tables from cache */
2351	ret = qcow2_cache_get(bs, s->l2_table_cache, slice_offset,
2352	(void **)&l2_slice);
2353	} else {
2354	/* load inactive L2 tables from disk */
2355	ret = bdrv_pread(bs->file, slice_offset, slice_size2,
2356	l2_slice, 0);
2357	}
2358	if (ret < 0) {
2359	goto fail;
2360	}
2361
2362	for (j = 0; j < s->l2_slice_size; j++) {
2363	uint64_t l2_entry = get_l2_entry(s, l2_slice, j);
2364	int64_t offset = l2_entry & L2E_OFFSET_MASK;
2365	QCow2ClusterType cluster_type =
2366	qcow2_get_cluster_type(bs, l2_entry);
2367
2368	if (cluster_type != QCOW2_CLUSTER_ZERO_PLAIN &&
2369	cluster_type != QCOW2_CLUSTER_ZERO_ALLOC) {
2370	continue;
2371	}
2372
2373	if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2374	if (!bs->backing) {
2375	/*
2376	* not backed; therefore we can simply deallocate the
2377	* cluster. No need to call set_l2_bitmap(), this
2378	* function doesn't support images with subclusters.
2379	*/
2380	set_l2_entry(s, l2_slice, j, 0);
2381	l2_dirty = true;
2382	continue;
2383	}
2384
2385	offset = qcow2_alloc_clusters(bs, s->cluster_size);
2386	if (offset < 0) {
2387	ret = offset;
2388	goto fail;
2389	}
2390
2391	/* The offset must fit in the offset field */
2392	assert((offset & L2E_OFFSET_MASK) == offset);
2393
2394	if (l2_refcount > 1) {
2395	/* For shared L2 tables, set the refcount accordingly
2396	* (it is already 1 and needs to be l2_refcount) */
2397	ret = qcow2_update_cluster_refcount(
2398	bs, offset >> s->cluster_bits,
2399	refcount_diff(1, l2_refcount), false,
2400	QCOW2_DISCARD_OTHER);
2401	if (ret < 0) {
2402	qcow2_free_clusters(bs, offset, s->cluster_size,
2403	QCOW2_DISCARD_OTHER);
2404	goto fail;
2405	}
2406	}
2407	}
2408
2409	if (offset_into_cluster(s, offset)) {
2410	int l2_index = slice * s->l2_slice_size + j;
2411	qcow2_signal_corruption(
2412	bs, true, -1, -1,
2413	"Cluster allocation offset "
2414	"%#" PRIx64 " unaligned (L2 offset: %#"
2415	PRIx64 ", L2 index: %#x)", offset,
2416	l2_offset, l2_index);
2417	if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2418	qcow2_free_clusters(bs, offset, s->cluster_size,
2419	QCOW2_DISCARD_ALWAYS);
2420	}
2421	ret = -EIO;
2422	goto fail;
2423	}
2424
2425	ret = qcow2_pre_write_overlap_check(bs, 0, offset,
2426	s->cluster_size, true);
2427	if (ret < 0) {
2428	if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2429	qcow2_free_clusters(bs, offset, s->cluster_size,
2430	QCOW2_DISCARD_ALWAYS);
2431	}
2432	goto fail;
2433	}
2434
2435	ret = bdrv_pwrite_zeroes(s->data_file, offset,
2436	s->cluster_size, 0);
2437	if (ret < 0) {
2438	if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2439	qcow2_free_clusters(bs, offset, s->cluster_size,
2440	QCOW2_DISCARD_ALWAYS);
2441	}
2442	goto fail;
2443	}
2444
2445	if (l2_refcount == 1) {
2446	set_l2_entry(s, l2_slice, j, offset \| QCOW_OFLAG_COPIED);
2447	} else {
2448	set_l2_entry(s, l2_slice, j, offset);
2449	}
2450	/*
2451	* No need to call set_l2_bitmap() after set_l2_entry() because
2452	* this function doesn't support images with subclusters.
2453	*/
2454	l2_dirty = true;
2455	}
2456
2457	if (is_active_l1) {
2458	if (l2_dirty) {
2459	qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
2460	qcow2_cache_depends_on_flush(s->l2_table_cache);
2461	}
2462	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2463	} else {
2464	if (l2_dirty) {
2465	ret = qcow2_pre_write_overlap_check(
2466	bs, QCOW2_OL_INACTIVE_L2 \| QCOW2_OL_ACTIVE_L2,
2467	slice_offset, slice_size2, false);
2468	if (ret < 0) {
2469	goto fail;
2470	}
2471
2472	ret = bdrv_pwrite(bs->file, slice_offset, slice_size2,
2473	l2_slice, 0);
2474	if (ret < 0) {
2475	goto fail;
2476	}
2477	}
2478	}
2479	}
2480
2481	(*visited_l1_entries)++;
2482	if (status_cb) {
2483	status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
2484	}
2485	}
2486
2487	ret = 0;
2488
2489	fail:
2490	if (l2_slice) {
2491	if (!is_active_l1) {
2492	qemu_vfree(l2_slice);
2493	} else {
2494	qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2495	}
2496	}
2497	return ret;
2498	}
2499
2500	/*
2501	* For backed images, expands all zero clusters on the image. For non-backed
2502	* images, deallocates all non-pre-allocated zero clusters (and claims the
2503	* allocation for pre-allocated ones). This is important for downgrading to a
2504	* qcow2 version which doesn't yet support metadata zero clusters.
2505	*/
2506	int qcow2_expand_zero_clusters(BlockDriverState *bs,
2507	BlockDriverAmendStatusCB *status_cb,
2508	void *cb_opaque)
2509	{
2510	BDRVQcow2State *s = bs->opaque;
2511	uint64_t *l1_table = NULL;
2512	int64_t l1_entries = 0, visited_l1_entries = 0;
2513	int ret;
2514	int i, j;
2515
2516	if (status_cb) {
2517	l1_entries = s->l1_size;
2518	for (i = 0; i < s->nb_snapshots; i++) {
2519	l1_entries += s->snapshots[i].l1_size;
2520	}
2521	}
2522
2523	ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
2524	&visited_l1_entries, l1_entries,
2525	status_cb, cb_opaque);
2526	if (ret < 0) {
2527	goto fail;
2528	}
2529
2530	/* Inactive L1 tables may point to active L2 tables - therefore it is
2531	* necessary to flush the L2 table cache before trying to access the L2
2532	* tables pointed to by inactive L1 entries (else we might try to expand
2533	* zero clusters that have already been expanded); furthermore, it is also
2534	* necessary to empty the L2 table cache, since it may contain tables which
2535	* are now going to be modified directly on disk, bypassing the cache.
2536	* qcow2_cache_empty() does both for us. */
2537	ret = qcow2_cache_empty(bs, s->l2_table_cache);
2538	if (ret < 0) {
2539	goto fail;
2540	}
2541
2542	for (i = 0; i < s->nb_snapshots; i++) {
2543	int l1_size2;
2544	uint64_t *new_l1_table;
2545	Error *local_err = NULL;
2546
2547	ret = qcow2_validate_table(bs, s->snapshots[i].l1_table_offset,
2548	s->snapshots[i].l1_size, L1E_SIZE,
2549	QCOW_MAX_L1_SIZE, "Snapshot L1 table",
2550	&local_err);
2551	if (ret < 0) {
2552	error_report_err(local_err);
2553	goto fail;
2554	}
2555
2556	l1_size2 = s->snapshots[i].l1_size * L1E_SIZE;
2557	new_l1_table = g_try_realloc(l1_table, l1_size2);
2558
2559	if (!new_l1_table) {
2560	ret = -ENOMEM;
2561	goto fail;
2562	}
2563
2564	l1_table = new_l1_table;
2565
2566	ret = bdrv_pread(bs->file, s->snapshots[i].l1_table_offset, l1_size2,
2567	l1_table, 0);
2568	if (ret < 0) {
2569	goto fail;
2570	}
2571
2572	for (j = 0; j < s->snapshots[i].l1_size; j++) {
2573	be64_to_cpus(&l1_table[j]);
2574	}
2575
2576	ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
2577	&visited_l1_entries, l1_entries,
2578	status_cb, cb_opaque);
2579	if (ret < 0) {
2580	goto fail;
2581	}
2582	}
2583
2584	ret = 0;
2585
2586	fail:
2587	g_free(l1_table);
2588	return ret;
2589	}
2590
2591	void qcow2_parse_compressed_l2_entry(BlockDriverState *bs, uint64_t l2_entry,
2592	uint64_t coffset, int csize)
2593	{
2594	BDRVQcow2State *s = bs->opaque;
2595	int nb_csectors;
2596
2597	assert(qcow2_get_cluster_type(bs, l2_entry) == QCOW2_CLUSTER_COMPRESSED);
2598
2599	*coffset = l2_entry & s->cluster_offset_mask;
2600
2601	nb_csectors = ((l2_entry >> s->csize_shift) & s->csize_mask) + 1;
2602	csize = nb_csectors QCOW2_COMPRESSED_SECTOR_SIZE -
2603	(*coffset & (QCOW2_COMPRESSED_SECTOR_SIZE - 1));
2604	}