index-pack: factor out unpack core from get_data_from_pack
This allows caller to consume large inflated object with a fixed amount of memory. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
May 23, 2012 at 21:09 UTC
8a2e163ccd03a12696cbf7518c0b742d313b6f02
1 file changed
+24
-3
builtin/index-pack.c
+24
-3
@@ -504,7 +504,9 @@ static void *unpack_raw_entry(struct object_entry *obj,
504
return data;
505
}
506
507
-static void *get_data_from_pack(struct object_entry *obj)
507
+static void *unpack_data(struct object_entry *obj,
508
+ int (*consume)(const unsigned char *, unsigned long, void *),
509
+ void *cb_data)
510
{
511
off_t from = obj[0].idx.offset + obj[0].hdr_size;
512
unsigned long len = obj[1].idx.offset - from;
@@ -512,15 +514,16 @@ static void *get_data_from_pack(struct object_entry *obj)
514
git_zstream stream;
515
int status;
516
515
- data = xmalloc(obj->size);
517
+ data = xmalloc(consume ? 64*1024 : obj->size);
518
inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
519
520
memset(&stream, 0, sizeof(stream));
521
git_inflate_init(&stream);
522
stream.next_out = data;
521
- stream.avail_out = obj->size;
523
+ stream.avail_out = consume ? 64*1024 : obj->size;
524
525
do {
526
+ unsigned char *last_out = stream.next_out;
527
ssize_t n = (len < 64*1024) ? len : 64*1024;
528
n = pread(pack_fd, inbuf, n, from);
529
if (n < 0)
@@ -535,6 +538,15 @@ static void *get_data_from_pack(struct object_entry *obj)
538
stream.next_in = inbuf;
539
stream.avail_in = n;
540
status = git_inflate(&stream, 0);
541
+ if (consume) {
542
+ if (consume(last_out, stream.next_out - last_out, cb_data)) {
543
+ free(inbuf);
544
+ free(data);
545
+ return NULL;
546
+ }
547
+ stream.next_out = data;
548
+ stream.avail_out = 64*1024;
549
+ }
550
} while (len && status == Z_OK && !stream.avail_in);
551
552
/* This has been inflated OK when first encountered, so... */
@@ -543,9 +555,18 @@ static void *get_data_from_pack(struct object_entry *obj)
555
556
git_inflate_end(&stream);
557
free(inbuf);
558
+ if (consume) {
559
+ free(data);
560
+ data = NULL;
561
+ }
562
return data;
563
}
564
565
+static void *get_data_from_pack(struct object_entry *obj)
566
+{
567
+ return unpack_data(obj, NULL, NULL);
568
+}
569
+
570
static int compare_delta_bases(const union delta_base *base1,
571
const union delta_base *base2,
572
enum object_type type1,