@matheusbsbs / matheus-bagliano / commits / 53dbce78a2
Fork 0

Map only part of the generated pack file at any point in time.

When generating a very large pack file (for example close to 1 GB in size) it may be impossible for the kernel to find a contiguous free range within a 32 bit address space for the mapping to be located at. This is especially problematic on large imports where there is a lot of malloc activity occuring within the same process and the malloc'd regions may straddle the previously mapped regions, thereby creating large holes in the address space. So instead we map only 128 MB of the pack at any given time. This will likely increase the number of times the file gets mapped (with additional system time required to update the page tables more frequently) but will allow the program to handle packs up to 4 GB in size. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 27, 2006 at 05:53 UTC 53dbce78a2a018bd2828d3ecc4123015f88ae36f
1 file changed +24 -20
fast-import.c
+24 -20
@@ -215,8 +215,10 @@ static struct atom_str **atom_table;
215 static int pack_fd;
216 static unsigned long pack_size;
217 static unsigned char pack_sha1[20];
218 -static void* pack_base;
219 -static size_t pack_mlen;
218 +static unsigned char* pack_base;
219 +static unsigned long pack_moff;
220 +static unsigned long pack_mlen = 128*1024*1024;
221 +static unsigned long page_size;
222
223 /* Table of objects we've written. */
224 static unsigned int object_entry_alloc = 1000;
@@ -676,23 +678,26 @@ static int store_object(
678 return 0;
679 }
680
679 -static void* map_pack(unsigned long offset)
681 +static unsigned char* map_pack(unsigned long offset, unsigned int *left)
682 {
683 if (offset >= pack_size)
684 die("object offset outside of pack file");
683 - if (offset >= pack_mlen) {
685 + if (!pack_base
686 + || offset < pack_moff
687 + || (offset + 20) >= (pack_moff + pack_mlen)) {
688 if (pack_base)
689 munmap(pack_base, pack_mlen);
686 - /* round out how much we map to 16 MB units */
687 - pack_mlen = pack_size;
688 - if (pack_mlen & ((1 << 24) - 1))
689 - pack_mlen = ((pack_mlen >> 24) + 1) << 24;
690 - pack_base = mmap(NULL,pack_mlen,PROT_READ,MAP_SHARED,pack_fd,0);
690 + pack_moff = (offset / page_size) * page_size;
691 + pack_base = mmap(NULL,pack_mlen,PROT_READ,MAP_SHARED,
692 + pack_fd,pack_moff);
693 if (pack_base == MAP_FAILED)
694 die("Failed to map generated pack: %s", strerror(errno));
695 remap_count++;
696 }
695 - return (char*)pack_base + offset;
697 + offset -= pack_moff;
698 + if (left)
699 + *left = pack_mlen - offset;
700 + return pack_base + offset;
701 }
702
703 static unsigned long unpack_object_header(unsigned long offset,
@@ -703,12 +708,12 @@ static unsigned long unpack_object_header(unsigned long offset,
708 unsigned char c;
709 unsigned long size;
710
706 - c = *(unsigned char*)map_pack(offset++);
711 + c = *map_pack(offset++, NULL);
712 *type = (c >> 4) & 7;
713 size = c & 15;
714 shift = 4;
715 while (c & 0x80) {
711 - c = *(unsigned char*)map_pack(offset++);
716 + c = *map_pack(offset++, NULL);
717 size += (c & 0x7f) << shift;
718 shift += 7;
719 }
@@ -725,8 +730,7 @@ static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
730 result[sz] = 0;
731
732 memset(&stream, 0, sizeof(stream));
728 - stream.next_in = map_pack(o);
729 - stream.avail_in = pack_mlen - o;
733 + stream.next_in = map_pack(o, &stream.avail_in);
734 stream.next_out = result;
735 stream.avail_out = sz;
736
@@ -735,13 +739,12 @@ static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
739 int st = inflate(&stream, Z_FINISH);
740 if (st == Z_STREAM_END)
741 break;
738 - if (st == Z_OK) {
739 - o = stream.next_in - (unsigned char*)pack_base;
740 - stream.next_in = map_pack(o);
741 - stream.avail_in = pack_mlen - o;
742 + if (st == Z_OK || st == Z_BUF_ERROR) {
743 + o = stream.next_in - pack_base + pack_moff;
744 + stream.next_in = map_pack(o, &stream.avail_in);
745 continue;
746 }
744 - die("Error from zlib during inflate.");
747 + die("Error %i from zlib during inflate.", st);
748 }
749 inflateEnd(&stream);
750 if (stream.total_out != sz)
@@ -760,7 +763,7 @@ static void *unpack_delta_entry(unsigned long offset,
763 void *delta_data, *base, *result;
764 unsigned long base_size, result_size;
765
763 - base_sha1 = (unsigned char*)map_pack(offset + 20) - 20;
766 + base_sha1 = map_pack(offset, NULL);
767 base_oe = find_object(base_sha1);
768 if (!base_oe)
769 die("I'm broken; I can't find a base I know must be here.");
@@ -1615,6 +1618,7 @@ int main(int argc, const char **argv)
1618
1619 setup_ident();
1620 git_config(git_default_config);
1621 + page_size = getpagesize();
1622
1623 for (i = 1; i < argc; i++) {
1624 const char *a = argv[i];