virtio-gpu: use g_try_malloc to avoid guest-triggered abort
Use g_try_malloc/g_try_new0 for guest-controlled allocation, so failure returns an error to the guest rather than crashing the host (glib behaviour). Resolves: https://gitlab.com/qemu-project/qemu/-/work_items/3898 Reviewed-by: Akihiko Odaki <odaki@rsg.ci.i.u-tokyo.ac.jp> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> Message-ID: <20260805130141.211398-1-marcandre.lureau@redhat.com>
Marc-André Lureau committed
Aug 5, 2026 at 17:01 UTC
eb2b3a1fa807f47515a1d86e434c60c981310bcd
7 files changed
+79
-26
contrib/vhost-user-gpu/vhost-user-gpu.c
+17
-8
@@ -487,7 +487,7 @@ vg_create_mapping_iov(VuGpu *g,
487
struct virtio_gpu_ctrl_command *cmd,
488
struct iovec **iov)
489
{
490
- struct virtio_gpu_mem_entry *ents;
490
+ g_autofree struct virtio_gpu_mem_entry *ents = NULL;
491
size_t esize, s;
492
int i;
493
@@ -498,17 +498,22 @@ vg_create_mapping_iov(VuGpu *g,
498
}
499
500
esize = sizeof(*ents) * ab->nr_entries;
501
- ents = g_malloc(esize);
501
+ ents = g_try_malloc(esize);
502
+ if (!ents && esize) {
503
+ return -1;
504
+ }
505
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
506
sizeof(*ab), ents, esize);
507
if (s != esize) {
508
g_critical("%s: command data size incorrect %zu vs %zu",
509
__func__, s, esize);
507
- g_free(ents);
510
return -1;
511
}
512
511
- *iov = g_new0(struct iovec, ab->nr_entries);
513
+ *iov = g_try_new0(struct iovec, ab->nr_entries);
514
+ if (!*iov && ab->nr_entries) {
515
+ return -1;
516
+ }
517
for (i = 0; i < ab->nr_entries; i++) {
518
uint64_t len = ents[i].length;
519
(*iov)[i].iov_len = ents[i].length;
@@ -517,12 +522,10 @@ vg_create_mapping_iov(VuGpu *g,
522
g_critical("%s: resource %d element %d",
523
__func__, ab->resource_id, i);
524
g_free(*iov);
520
- g_free(ents);
525
*iov = NULL;
526
return -1;
527
}
528
}
525
- g_free(ents);
529
return 0;
530
}
531
@@ -828,8 +831,14 @@ vg_resource_flush(VuGpu *g,
831
PIXMAN_FORMAT_BPP(pixman_image_get_format(res->image)) / 8;
832
size_t size = width * height * bpp;
833
831
- void *p = g_malloc(VHOST_USER_GPU_HDR_SIZE +
832
- sizeof(VhostUserGpuUpdate) + size);
834
+ void *p = g_try_malloc(VHOST_USER_GPU_HDR_SIZE +
835
+ sizeof(VhostUserGpuUpdate) + size);
836
+ if (!p) {
837
+ pixman_region_fini(®ion);
838
+ pixman_region_fini(&finalregion);
839
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
840
+ break;
841
+ }
842
VhostUserGpuMsg *msg = p;
843
msg->request = VHOST_USER_GPU_UPDATE;
844
msg->size = sizeof(VhostUserGpuUpdate) + size;
contrib/vhost-user-gpu/virgl.c
+5
-1
@@ -209,7 +209,11 @@ virgl_cmd_submit_3d(VuGpu *g,
209
return;
210
}
211
212
- buf = g_malloc(cs.size);
212
+ buf = g_try_malloc(cs.size);
213
+ if (!buf && cs.size) {
214
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
215
+ return;
216
+ }
217
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
218
sizeof(cs), buf, cs.size);
219
if (s != cs.size) {
contrib/vhost-user-gpu/vugbm.c
+4
-1
@@ -13,7 +13,10 @@
13
static bool
14
mem_alloc_bo(struct vugbm_buffer *buf)
15
{
16
- buf->mmap = g_malloc((uint64_t)buf->width * buf->height * 4);
16
+ buf->mmap = g_try_malloc((uint64_t)buf->width * buf->height * 4);
17
+ if (!buf->mmap && buf->width && buf->height) {
18
+ return false;
19
+ }
20
buf->stride = buf->width * 4;
21
return true;
22
}
hw/display/virtio-gpu-rutabaga.c
+12
-2
@@ -366,10 +366,20 @@ rutabaga_cmd_submit_3d(VirtIOGPU *g,
366
return;
367
}
368
369
- buf = g_new0(uint8_t, cs.size);
369
+ buf = g_try_new0(uint8_t, cs.size);
370
+ if (!buf && cs.size) {
371
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
372
+ return;
373
+ }
374
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
375
sizeof(cs), buf, cs.size);
372
- CHECK(s == cs.size, cmd);
376
+ if (s != cs.size) {
377
+ qemu_log_mask(LOG_GUEST_ERROR,
378
+ "%s: size mismatch (%zu/%u)\n",
379
+ __func__, s, cs.size);
380
+ cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
381
+ return;
382
+ }
383
384
rutabaga_cmd.ctx_id = cs.hdr.ctx_id;
385
rutabaga_cmd.cmd = buf;
hw/display/virtio-gpu-udmabuf.c
+5
-2
@@ -39,8 +39,11 @@ static void virtio_gpu_create_udmabuf(struct virtio_gpu_simple_resource *res)
39
return;
40
}
41
42
- list = g_malloc0(sizeof(struct udmabuf_create_list) +
43
- sizeof(struct udmabuf_create_item) * res->iov_cnt);
42
+ list = g_try_malloc0(sizeof(struct udmabuf_create_list) +
43
+ sizeof(struct udmabuf_create_item) * res->iov_cnt);
44
+ if (!list) {
45
+ return;
46
+ }
47
48
for (i = 0; i < res->iov_cnt; i++) {
49
rcu_read_lock();
hw/display/virtio-gpu-virgl.c
+5
-1
@@ -620,7 +620,11 @@ static void virgl_cmd_submit_3d(VirtIOGPU *g,
620
return;
621
}
622
623
- buf = g_malloc(cs.size);
623
+ buf = g_try_malloc(cs.size);
624
+ if (!buf && cs.size) {
625
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
626
+ return;
627
+ }
628
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
629
sizeof(cs), buf, cs.size);
630
if (s != cs.size) {
hw/display/virtio-gpu.c
+31
-11
@@ -892,7 +892,10 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
892
}
893
894
esize = sizeof(*ents) * nr_entries;
895
- ents = g_malloc(esize);
895
+ ents = g_try_malloc(esize);
896
+ if (!ents && esize) {
897
+ return -1;
898
+ }
899
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
900
offset, ents, esize);
901
if (s != esize) {
@@ -913,6 +916,7 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
916
hwaddr len;
917
void *map;
918
919
+ /* TODO: a common DMA map SG helper */
920
do {
921
len = l;
922
map = dma_memory_map(VIRTIO_DEVICE(g)->dma_as, a, &len,
@@ -921,20 +925,27 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
925
if (!map) {
926
qemu_log_mask(LOG_GUEST_ERROR, "%s: failed to map MMIO memory for"
927
" element %d\n", __func__, e);
924
- virtio_gpu_cleanup_mapping_iov(g, *iov, v);
925
- g_free(ents);
926
- *iov = NULL;
927
- if (addr) {
928
- g_free(*addr);
929
- *addr = NULL;
930
- }
931
- return -1;
928
+ goto err;
929
}
930
931
if (!(v % 16)) {
935
- *iov = g_renew(struct iovec, *iov, v + 16);
932
+ struct iovec *new_iov;
933
+ new_iov = g_try_renew(struct iovec, *iov, v + 16);
934
+ if (!new_iov) {
935
+ dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
936
+ DMA_DIRECTION_TO_DEVICE, len);
937
+ goto err;
938
+ }
939
+ *iov = new_iov;
940
if (addr) {
937
- *addr = g_renew(uint64_t, *addr, v + 16);
941
+ uint64_t *new_addr;
942
+ new_addr = g_try_renew(uint64_t, *addr, v + 16);
943
+ if (!new_addr) {
944
+ dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
945
+ DMA_DIRECTION_TO_DEVICE, len);
946
+ goto err;
947
+ }
948
+ *addr = new_addr;
949
}
950
}
951
(*iov)[v].iov_base = map;
@@ -952,6 +963,15 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
963
964
g_free(ents);
965
return 0;
966
+
967
+err:
968
+ virtio_gpu_cleanup_mapping_iov(g, *iov, v);
969
+ *iov = NULL;
970
+ if (addr) {
971
+ g_clear_pointer(addr, g_free);
972
+ }
973
+ g_free(ents);
974
+ return -1;
975
}
976
977
void virtio_gpu_cleanup_mapping_iov(VirtIOGPU *g,