zlib: allow feeding more than 4GB in one go
Update zlib_post_call() that adjusts the wrapper's notion of avail_in and avail_out to what came back from zlib, so that the callers can feed buffers larger than than 4GB to the API. When underlying inflate/deflate stopped processing because we fed a buffer larger than 4GB limit, detect that case, update the state variables, and let the zlib function work another round. Signed-off-by: Junio C Hamano <gitster@pobox.com>
Junio C Hamano committed
Jun 10, 2011 at 12:15 UTC
e01503b523e79748ac91d876f506811c597d03cb
1 file changed
+58
-20
zlib.c
+58
-20
@@ -27,12 +27,11 @@ static const char *zerr_to_string(int status)
27
* limits the size of the buffer we can use to 4GB when interacting
28
* with zlib in a single call to inflate/deflate.
29
*/
30
-#define ZLIB_BUF_MAX ((uInt)-1)
30
+/* #define ZLIB_BUF_MAX ((uInt)-1) */
31
+#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */
32
static inline uInt zlib_buf_cap(unsigned long len)
33
{
33
- if (ZLIB_BUF_MAX < len)
34
- die("working buffer for zlib too large");
35
- return len;
34
+ return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len;
35
}
36
37
static void zlib_pre_call(git_zstream *s)
@@ -47,12 +46,22 @@ static void zlib_pre_call(git_zstream *s)
46
47
static void zlib_post_call(git_zstream *s)
48
{
49
+ unsigned long bytes_consumed;
50
+ unsigned long bytes_produced;
51
+
52
+ bytes_consumed = s->z.next_in - s->next_in;
53
+ bytes_produced = s->z.next_out - s->next_out;
54
+ if (s->z.total_out != s->total_out + bytes_produced)
55
+ die("BUG: total_out mismatch");
56
+ if (s->z.total_in != s->total_in + bytes_consumed)
57
+ die("BUG: total_in mismatch");
58
+
59
+ s->total_out = s->z.total_out;
60
+ s->total_in = s->z.total_in;
61
s->next_in = s->z.next_in;
62
s->next_out = s->z.next_out;
52
- s->total_in = s->z.total_in;
53
- s->total_out = s->z.total_out;
54
- s->avail_in = s->z.avail_in;
55
- s->avail_out = s->z.avail_out;
63
+ s->avail_in -= bytes_consumed;
64
+ s->avail_out -= bytes_produced;
65
}
66
67
void git_inflate_init(git_zstream *strm)
@@ -103,18 +112,32 @@ int git_inflate(git_zstream *strm, int flush)
112
{
113
int status;
114
106
- zlib_pre_call(strm);
107
- status = inflate(&strm->z, flush);
108
- zlib_post_call(strm);
115
+ for (;;) {
116
+ zlib_pre_call(strm);
117
+ /* Never say Z_FINISH unless we are feeding everything */
118
+ status = inflate(&strm->z,
119
+ (strm->z.avail_in != strm->avail_in)
120
+ ? 0 : flush);
121
+ if (status == Z_MEM_ERROR)
122
+ die("inflate: out of memory");
123
+ zlib_post_call(strm);
124
+
125
+ /*
126
+ * Let zlib work another round, while we can still
127
+ * make progress.
128
+ */
129
+ if ((strm->avail_out && !strm->z.avail_out) &&
130
+ (status == Z_OK || status == Z_BUF_ERROR))
131
+ continue;
132
+ break;
133
+ }
134
+
135
switch (status) {
136
/* Z_BUF_ERROR: normal, needs more space in the output buffer */
137
case Z_BUF_ERROR:
138
case Z_OK:
139
case Z_STREAM_END:
140
return status;
115
-
116
- case Z_MEM_ERROR:
117
- die("inflate: out of memory");
141
default:
142
break;
143
}
@@ -192,18 +215,33 @@ int git_deflate(git_zstream *strm, int flush)
215
{
216
int status;
217
195
- zlib_pre_call(strm);
196
- status = deflate(&strm->z, flush);
197
- zlib_post_call(strm);
218
+ for (;;) {
219
+ zlib_pre_call(strm);
220
+
221
+ /* Never say Z_FINISH unless we are feeding everything */
222
+ status = deflate(&strm->z,
223
+ (strm->z.avail_in != strm->avail_in)
224
+ ? 0 : flush);
225
+ if (status == Z_MEM_ERROR)
226
+ die("deflate: out of memory");
227
+ zlib_post_call(strm);
228
+
229
+ /*
230
+ * Let zlib work another round, while we can still
231
+ * make progress.
232
+ */
233
+ if ((strm->avail_out && !strm->z.avail_out) &&
234
+ (status == Z_OK || status == Z_BUF_ERROR))
235
+ continue;
236
+ break;
237
+ }
238
+
239
switch (status) {
240
/* Z_BUF_ERROR: normal, needs more space in the output buffer */
241
case Z_BUF_ERROR:
242
case Z_OK:
243
case Z_STREAM_END:
244
return status;
204
-
205
- case Z_MEM_ERROR:
206
- die("deflate: out of memory");
245
default:
246
break;
247
}