hw/display/sm501: Avoid overflow problems in bounds check calculations
When we check that a 2D rectangle operation isn't going to run off the end of video RAM, we do the calculations as 32 bit arithmetic. This means that carefully chosen guest register values can cause an overflow so we don't detect that the operation is going to go outside video memory. Abstract the check out into a function, do the calculations as 64-bit arithmetic, and add assertions about the ranges of the inputs. Cc: qemu-stable@nongnu.org Resolves: https://gitlab.com/qemu-project/qemu/-/work_items/3584 Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Tested-by: BALATON Zoltan <balaton@eik.bme.hu> Reviewed-by: BALATON Zoltan <balaton@eik.bme.hu> Message-id: 20260707150933.1410507-3-peter.maydell@linaro.org
Peter Maydell committed
Jul 13, 2026 at 12:34 UTC
422734d5fc2b5adec704b352082385511959d5a7
1 file changed
+26
-6
hw/display/sm501.c
+26
-6
@@ -682,6 +682,28 @@ static inline void hwc_invalidate(SM501State *s, int crt)
682
get_fb_addr(s, crt) + start, end - start);
683
}
684
685
+static bool sm501_rect_outside_vram(SM501State *s, uint32_t base,
686
+ uint32_t x, uint32_t y,
687
+ uint32_t width, uint32_t height,
688
+ uint32_t pitch, uint32_t bypp)
689
+{
690
+ /*
691
+ * Return true if the 2D area specified by the arguments is
692
+ * partially or completely outside the VRAM (a guest error)
693
+ *
694
+ * Limits on the input sizes mean we can't overflow as long as
695
+ * we do all the arithmetic at 64 bits.
696
+ */
697
+ uint64_t rect_size, last_addr;
698
+
699
+ assert(x <= UINT16_MAX && y <= UINT16_MAX && height <= UINT16_MAX &&
700
+ pitch <= UINT16_MAX && bypp <= 8);
701
+ rect_size = (((uint64_t)y + height) * pitch + x + width) * bypp;
702
+ last_addr = base + rect_size;
703
+
704
+ return last_addr >= get_local_mem_size(s);
705
+}
706
+
707
static void sm501_2d_operation(SM501State *s)
708
{
709
int cmd = (s->twoD_control >> 16) & 0x1F;
@@ -731,9 +753,8 @@ static void sm501_2d_operation(SM501State *s)
753
dst_y -= height - 1;
754
}
755
734
- if (dst_base >= get_local_mem_size(s) ||
735
- dst_base + (dst_x + width + (dst_y + height) * dst_pitch) * bypp >=
736
- get_local_mem_size(s)) {
756
+ if (sm501_rect_outside_vram(s, dst_base, dst_x, dst_y, width, height,
757
+ dst_pitch, bypp)) {
758
qemu_log_mask(LOG_GUEST_ERROR, "sm501: 2D op dest is outside vram.\n");
759
return;
760
}
@@ -760,9 +781,8 @@ static void sm501_2d_operation(SM501State *s)
781
src_y -= height - 1;
782
}
783
763
- if (src_base >= get_local_mem_size(s) ||
764
- src_base + (src_x + width + (src_y + height) * src_pitch) * bypp >=
765
- get_local_mem_size(s)) {
784
+ if (sm501_rect_outside_vram(s, src_base, src_x, src_y, width, height,
785
+ src_pitch, bypp)) {
786
qemu_log_mask(LOG_GUEST_ERROR,
787
"sm501: 2D op src is outside vram.\n");
788
return;