index-pack: use streaming interface on large blobs (most of the time)

unpack_raw_entry() will not allocate and return decompressed blobs if they are larger than core.bigFileThreshold. sha1_object() may not be called on those objects because there's no actual content. sha1_object() is called later on those objects, where we can safely use get_data_from_pack() to retrieve blob content for checking. However we always do that when we definitely need the blob content. And we often don't. There are two cases when we may need object content. The first case is when we find an in-repo blob with the same SHA-1. We need to do collision test, byte-on-byte. If this test is on, the blob must be loaded on memory (i.e. no streaming). Normally (e.g. in fetch/pull/clone) this does not happen because git avoid to send objects that client already has. The other case is when --strict is specified and the object in question is not a blob, which can't happen in reality becase we deal with large _blobs_ here. Note: --verify (or git-verify-pack) a pack from current repository will trigger collision test on every object in the pack, which effectively disables this patch. This could be easily worked around by setting GIT_DIR to an imaginary place with no packs. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nguyễn Thái Ngọc Duy committed May 23, 2012 at 21:09 UTC 9ec2dde9f3008ed3822c8e7510ab710bdb613873
2 files changed +49 -8
builtin/index-pack.c
+44 -8
@@ -392,9 +392,10 @@ static int is_delta_type(enum object_type type)
392 static void *unpack_entry_data(unsigned long offset, unsigned long size,
393 enum object_type type, unsigned char *sha1)
394 {
395 + static char fixed_buf[8192];
396 int status;
397 git_zstream stream;
397 - void *buf = xmalloc(size);
398 + void *buf;
399 git_SHA_CTX c;
400 char hdr[32];
401 int hdrlen;
@@ -405,11 +406,15 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,
406 git_SHA1_Update(&c, hdr, hdrlen);
407 } else
408 sha1 = NULL;
409 + if (type == OBJ_BLOB && size > big_file_threshold)
410 + buf = fixed_buf;
411 + else
412 + buf = xmalloc(size);
413
414 memset(&stream, 0, sizeof(stream));
415 git_inflate_init(&stream);
416 stream.next_out = buf;
412 - stream.avail_out = size;
417 + stream.avail_out = buf == fixed_buf ? sizeof(fixed_buf) : size;
418
419 do {
420 unsigned char *last_out = stream.next_out;
@@ -419,13 +424,17 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,
424 use(input_len - stream.avail_in);
425 if (sha1)
426 git_SHA1_Update(&c, last_out, stream.next_out - last_out);
427 + if (buf == fixed_buf) {
428 + stream.next_out = buf;
429 + stream.avail_out = sizeof(fixed_buf);
430 + }
431 } while (status == Z_OK);
432 if (stream.total_out != size || status != Z_STREAM_END)
433 bad_object(offset, _("inflate returned %d"), status);
434 git_inflate_end(&stream);
435 if (sha1)
436 git_SHA1_Final(sha1, &c);
428 - return buf;
437 + return buf == fixed_buf ? NULL : buf;
438 }
439
440 static void *unpack_raw_entry(struct object_entry *obj,
@@ -591,14 +600,21 @@ static void find_delta_children(const union delta_base *base,
600 *last_index = last;
601 }
602
594 -static void sha1_object(const void *data, unsigned long size,
595 - enum object_type type, const unsigned char *sha1)
603 +static void sha1_object(const void *data, struct object_entry *obj_entry,
604 + unsigned long size, enum object_type type,
605 + const unsigned char *sha1)
606 {
607 + void *new_data = NULL;
608 +
609 + assert(data || obj_entry);
610 +
611 read_lock();
612 if (has_sha1_file(sha1)) {
613 void *has_data;
614 enum object_type has_type;
615 unsigned long has_size;
616 + if (!data)
617 + data = new_data = get_data_from_pack(obj_entry);
618 has_data = read_sha1_file(sha1, &has_type, &has_size);
619 read_unlock();
620 if (!has_data)
@@ -623,6 +639,9 @@ static void sha1_object(const void *data, unsigned long size,
639 int eaten;
640 void *buf = (void *) data;
641
642 + if (!buf)
643 + buf = new_data = get_data_from_pack(obj_entry);
644 +
645 /*
646 * we do not need to free the memory here, as the
647 * buf is deleted by the caller.
@@ -647,6 +666,8 @@ static void sha1_object(const void *data, unsigned long size,
666 }
667 read_unlock();
668 }
669 +
670 + free(new_data);
671 }
672
673 /*
@@ -730,7 +751,7 @@ static void resolve_delta(struct object_entry *delta_obj,
751 bad_object(delta_obj->idx.offset, _("failed to apply delta"));
752 hash_sha1_file(result->data, result->size,
753 typename(delta_obj->real_type), delta_obj->idx.sha1);
733 - sha1_object(result->data, result->size, delta_obj->real_type,
754 + sha1_object(result->data, NULL, result->size, delta_obj->real_type,
755 delta_obj->idx.sha1);
756 counter_lock();
757 nr_resolved_deltas++;
@@ -860,7 +881,7 @@ static void *threaded_second_pass(void *data)
881 */
882 static void parse_pack_objects(unsigned char *sha1)
883 {
863 - int i;
884 + int i, nr_delays = 0;
885 struct delta_entry *delta = deltas;
886 struct stat st;
887
@@ -876,8 +897,12 @@ static void parse_pack_objects(unsigned char *sha1)
897 nr_deltas++;
898 delta->obj_no = i;
899 delta++;
900 + } else if (!data) {
901 + /* large blobs, check later */
902 + obj->real_type = OBJ_BAD;
903 + nr_delays++;
904 } else
880 - sha1_object(data, obj->size, obj->type, obj->idx.sha1);
905 + sha1_object(data, NULL, obj->size, obj->type, obj->idx.sha1);
906 free(data);
907 display_progress(progress, i+1);
908 }
@@ -897,6 +922,17 @@ static void parse_pack_objects(unsigned char *sha1)
922 if (S_ISREG(st.st_mode) &&
923 lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)
924 die(_("pack has junk at the end"));
925 +
926 + for (i = 0; i < nr_objects; i++) {
927 + struct object_entry *obj = &objects[i];
928 + if (obj->real_type != OBJ_BAD)
929 + continue;
930 + obj->real_type = obj->type;
931 + sha1_object(NULL, obj, obj->size, obj->type, obj->idx.sha1);
932 + nr_delays--;
933 + }
934 + if (nr_delays)
935 + die(_("confusion beyond insanity in parse_pack_objects()"));
936 }
937
938 /*
t/t1050-large.sh
+5
@@ -130,6 +130,11 @@ test_expect_success 'git-show a large file' '
130
131 '
132
133 +test_expect_success 'index-pack' '
134 + git clone file://"`pwd`"/.git foo &&
135 + GIT_DIR=non-existent git index-pack --strict --verify foo/.git/objects/pack/*.pack
136 +'
137 +
138 test_expect_success 'repack' '
139 git repack -ad
140 '