git-zlib: handle data streams larger than 4GB
On Windows, zlib's `uLong` type is 32-bit even on 64-bit systems. When processing data streams larger than 4GB, the `total_in` and `total_out` fields in zlib's `z_stream` structure wrap around, which caused the sanity checks in `zlib_post_call()` to trigger `BUG()` assertions. The git_zstream wrapper now tracks its own 64-bit totals rather than copying them from zlib. The sanity checks compare only the low bits, using `maximum_unsigned_value_of_type(uLong)` to mask appropriately for the platform's `uLong` size. This is based on work by LordKiRon in git-for-windows#6076. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Johannes Schindelin committed
May 8, 2026 at 08:16 UTC
d05d6669778fa7dbd72f37c3ad9e4024ca8693d1
3 files changed
+20
-11
git-zlib.c
+17
-8
@@ -30,6 +30,9 @@ static const char *zerr_to_string(int status)
30
*/
31
/* #define ZLIB_BUF_MAX ((uInt)-1) */
32
#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */
33
+
34
+/* uLong is 32-bit on Windows, even on 64-bit systems */
35
+#define ULONG_MAX_VALUE maximum_unsigned_value_of_type(uLong)
36
static inline uInt zlib_buf_cap(unsigned long len)
37
{
38
return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len;
@@ -39,31 +42,37 @@ static void zlib_pre_call(git_zstream *s)
42
{
43
s->z.next_in = s->next_in;
44
s->z.next_out = s->next_out;
42
- s->z.total_in = s->total_in;
43
- s->z.total_out = s->total_out;
45
+ s->z.total_in = (uLong)(s->total_in & ULONG_MAX_VALUE);
46
+ s->z.total_out = (uLong)(s->total_out & ULONG_MAX_VALUE);
47
s->z.avail_in = zlib_buf_cap(s->avail_in);
48
s->z.avail_out = zlib_buf_cap(s->avail_out);
49
}
50
51
static void zlib_post_call(git_zstream *s, int status)
52
{
50
- unsigned long bytes_consumed;
51
- unsigned long bytes_produced;
53
+ size_t bytes_consumed;
54
+ size_t bytes_produced;
55
56
bytes_consumed = s->z.next_in - s->next_in;
57
bytes_produced = s->z.next_out - s->next_out;
55
- if (s->z.total_out != s->total_out + bytes_produced)
58
+ /*
59
+ * zlib's total_out/total_in are uLong which may wrap for >4GB.
60
+ * We track our own totals and verify only the low bits match.
61
+ */
62
+ if ((s->z.total_out & ULONG_MAX_VALUE) !=
63
+ ((s->total_out + bytes_produced) & ULONG_MAX_VALUE))
64
BUG("total_out mismatch");
65
/*
66
* zlib does not update total_in when it returns Z_NEED_DICT,
67
* causing a mismatch here. Skip the sanity check in that case.
68
*/
69
if (status != Z_NEED_DICT &&
62
- s->z.total_in != s->total_in + bytes_consumed)
70
+ (s->z.total_in & ULONG_MAX_VALUE) !=
71
+ ((s->total_in + bytes_consumed) & ULONG_MAX_VALUE))
72
BUG("total_in mismatch");
73
65
- s->total_out = s->z.total_out;
66
- s->total_in = s->z.total_in;
74
+ s->total_out += bytes_produced;
75
+ s->total_in += bytes_consumed;
76
/* zlib-ng marks `next_in` as `const`, so we have to cast it away. */
77
s->next_in = (unsigned char *) s->z.next_in;
78
s->next_out = s->z.next_out;
git-zlib.h
+2
-2
@@ -7,8 +7,8 @@ typedef struct git_zstream {
7
struct z_stream_s z;
8
unsigned long avail_in;
9
unsigned long avail_out;
10
- unsigned long total_in;
11
- unsigned long total_out;
10
+ size_t total_in;
11
+ size_t total_out;
12
unsigned char *next_in;
13
unsigned char *next_out;
14
} git_zstream;
object-file.c
+1
-1
@@ -1118,7 +1118,7 @@ int odb_source_loose_write_stream(struct odb_source *source,
1118
} while (ret == Z_OK || ret == Z_BUF_ERROR);
1119
1120
if (stream.total_in != len + hdrlen)
1121
- die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
1121
+ die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
1122
(uintmax_t)len + hdrlen);
1123
1124
/*