index-pack: smarter memory usage when resolving deltas
In the same spirit as commit 9892bebafe, let's avoid allocating the full buffer for the deflated data in get_data_from_pack() in order to inflate it. Let's read and inflate the data in chunks instead to reduce memory usage. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nicolas Pitre committed
Apr 12, 2010 at 12:11 UTC
776ea3707a0248b9d36d696ee92beeeac5ccf1a1
1 file changed
+25
-21
builtin-index-pack.c
+25
-21
@@ -359,34 +359,38 @@ static void *get_data_from_pack(struct object_entry *obj)
359
{
360
off_t from = obj[0].idx.offset + obj[0].hdr_size;
361
unsigned long len = obj[1].idx.offset - from;
362
- unsigned long rdy = 0;
363
- unsigned char *src, *data;
362
+ unsigned char *data, *inbuf;
363
z_stream stream;
365
- int st;
364
+ int status;
365
367
- src = xmalloc(len);
368
- data = src;
369
- do {
370
- ssize_t n = pread(pack_fd, data + rdy, len - rdy, from + rdy);
371
- if (n < 0)
372
- die_errno("cannot pread pack file");
373
- if (!n)
374
- die("premature end of pack file, %lu bytes missing",
375
- len - rdy);
376
- rdy += n;
377
- } while (rdy < len);
366
data = xmalloc(obj->size);
367
+ inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
368
+
369
memset(&stream, 0, sizeof(stream));
370
+ git_inflate_init(&stream);
371
stream.next_out = data;
372
stream.avail_out = obj->size;
382
- stream.next_in = src;
383
- stream.avail_in = len;
384
- git_inflate_init(&stream);
385
- while ((st = git_inflate(&stream, Z_FINISH)) == Z_OK);
386
- git_inflate_end(&stream);
387
- if (st != Z_STREAM_END || stream.total_out != obj->size)
373
+
374
+ do {
375
+ ssize_t n = (len < 64*1024) ? len : 64*1024;
376
+ n = pread(pack_fd, inbuf, n, from);
377
+ if (n < 0)
378
+ die_errno("cannot pread pack file");
379
+ if (!n)
380
+ die("premature end of pack file, %lu bytes missing", len);
381
+ from += n;
382
+ len -= n;
383
+ stream.next_in = inbuf;
384
+ stream.avail_in = n;
385
+ status = git_inflate(&stream, 0);
386
+ } while (len && status == Z_OK && !stream.avail_in);
387
+
388
+ /* This has been inflated OK when first encountered, so... */
389
+ if (status != Z_STREAM_END || stream.total_out != obj->size)
390
die("serious inflate inconsistency");
389
- free(src);
391
+
392
+ git_inflate_end(&stream);
393
+ free(inbuf);
394
return data;
395
}
396