ui/pixman: fix zero rowstride in qemu_pixman_image_new_shareable()
qemu_create_displaysurface_from() callers such as xlnx_dp.c pass linesize=0 with data=NULL, relying on pixman to compute the stride. Since 1ff788db978 ("ui: use a shareable type"), the data=NULL path goes through qemu_pixman_image_new_shareable() which computes size = height * rowstride_bytes, resulting in a zero-size allocation and an abort in qemu_memfd_alloc(). Introduce qemu-pixman-helpers.h with overflow-safe stride and buffer size computation (matching pixman's create_bits() formula), and use it from both qemu_pixman_image_new_shareable() and pixman-minimal's create_bits(). Reported-by: Peter Maydell <peter.maydell@linaro.org> Fixes: 1ff788db9781 ("ui: use a shareable type") Reviewed-by: Philippe Mathieu-Daudé <philmd@oss.qualcomm.com> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-ID: <20260611113614.1935094-1-marcandre.lureau@redhat.com>
Marc-André Lureau committed
Jun 11, 2026 at 15:36 UTC
f7add4533d6727a6dd3d000d437e9b9730895ad0
4 files changed
+61
-22
include/ui/pixman-minimal.h
+4
-21
@@ -113,6 +113,8 @@ typedef struct pixman_color {
113
uint16_t alpha;
114
} pixman_color_t;
115
116
+#include "qemu-pixman-helpers.h"
117
+
118
static inline uint32_t *create_bits(pixman_format_code_t format,
119
int width,
120
int height,
@@ -120,28 +122,9 @@ static inline uint32_t *create_bits(pixman_format_code_t format,
122
{
123
int stride = 0;
124
size_t buf_size = 0;
123
- int bpp = PIXMAN_FORMAT_BPP(format);
124
-
125
- /*
126
- * Calculate the following while checking for overflow truncation:
127
- * stride = ((width * bpp + 0x1f) >> 5) * sizeof(uint32_t);
128
- */
129
-
130
- if (unlikely(__builtin_mul_overflow(width, bpp, &stride))) {
131
- return NULL;
132
- }
133
-
134
- if (unlikely(__builtin_add_overflow(stride, 0x1f, &stride))) {
135
- return NULL;
136
- }
137
-
138
- stride >>= 5;
139
-
140
- stride *= sizeof(uint32_t);
125
142
- if (unlikely(__builtin_mul_overflow((size_t) height,
143
- (size_t) stride,
144
- &buf_size))) {
126
+ if (!qemu_pixman_image_calc_size(format, width, height,
127
+ &stride, &buf_size)) {
128
return NULL;
129
}
130
include/ui/qemu-pixman-helpers.h
new
+49
@@ -0,0 +1,49 @@
1
+/* SPDX-License-Identifier: MIT */
2
+/*
3
+ * Pixman stride and buffer size helpers.
4
+ * Expects PIXMAN_FORMAT_BPP() and pixman_format_code_t to be
5
+ * already defined (by either <pixman.h> or "pixman-minimal.h").
6
+ */
7
+
8
+#ifndef QEMU_PIXMAN_HELPERS_H
9
+#define QEMU_PIXMAN_HELPERS_H
10
+
11
+/*
12
+ * Compute the row stride for a pixman image, aligned to sizeof(uint32_t),
13
+ * as pixman does. Returns -1 on integer overflow.
14
+ */
15
+static inline int qemu_pixman_stride(pixman_format_code_t format, int width)
16
+{
17
+ int stride;
18
+
19
+ if (unlikely(__builtin_mul_overflow(width, PIXMAN_FORMAT_BPP(format),
20
+ &stride)) ||
21
+ unlikely(__builtin_add_overflow(stride, 31, &stride))) {
22
+ return -1;
23
+ }
24
+ return (stride / 32) * sizeof(uint32_t);
25
+}
26
+
27
+/*
28
+ * Compute stride and buffer size for a pixman image.
29
+ * If *rowstride_bytes is 0, compute it from format and width
30
+ * (aligned to sizeof(uint32_t), as pixman does).
31
+ * Returns false on integer overflow.
32
+ */
33
+static inline bool qemu_pixman_image_calc_size(pixman_format_code_t format,
34
+ int width, int height,
35
+ int *rowstride_bytes,
36
+ size_t *buf_size)
37
+{
38
+ if (!*rowstride_bytes) {
39
+ *rowstride_bytes = qemu_pixman_stride(format, width);
40
+ if (*rowstride_bytes < 0) {
41
+ return false;
42
+ }
43
+ }
44
+
45
+ return likely(!__builtin_mul_overflow((size_t)height,
46
+ (size_t)*rowstride_bytes, buf_size));
47
+}
48
+
49
+#endif /* QEMU_PIXMAN_HELPERS_H */
include/ui/qemu-pixman.h
+1
@@ -8,6 +8,7 @@
8
9
#ifdef CONFIG_PIXMAN
10
#include <pixman.h>
11
+#include "qemu-pixman-helpers.h"
12
#else
13
#include "pixman-minimal.h"
14
#endif
ui/qemu-pixman.c
+7
-1
@@ -320,12 +320,18 @@ qemu_pixman_image_new_shareable(pixman_image_t **image,
320
Error **errp)
321
{
322
ERRP_GUARD();
323
- size_t size = height * rowstride_bytes;
323
+ size_t size;
324
void *bits = NULL;
325
326
g_return_val_if_fail(image != NULL, false);
327
g_return_val_if_fail(handle != NULL, false);
328
329
+ if (!qemu_pixman_image_calc_size(format, width, height,
330
+ &rowstride_bytes, &size)) {
331
+ error_setg(errp, "Image dimensions overflow");
332
+ return false;
333
+ }
334
+
335
bits = qemu_pixman_shareable_alloc(name, size, handle, errp);
336
if (!bits) {
337
return false;