@matheusbsbs / matheus-bagliano / commits / f70b653429
Fork 0

Restructure fast-import to support creating multiple packfiles.

Now that we are starting to see some really large projects (such as KDE or a fork of FreeBSD) get imported into Git we're running into the upper limit on packfile object count as well as overall byte length. The KDE and FreeBSD projects are both likely to require more than 4 GiB to store their current history, which means we really need multiple packfiles to handle their content. This is a fairly simple restructuring of the internal code to help us support creating multiple packfiles from within fast-import. We are now adding a 5 digit incrementing suffix to the end of the basename supplied to us by the caller, permitting up to 99,999 packs to be generated in a single fast-import run. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Jan 15, 2007 at 04:39 UTC f70b653429ebc7fdde0b36a63e1deb4aadb450d3
1 file changed +124 -121
fast-import.c
+124 -121
@@ -216,7 +216,6 @@ static unsigned long max_depth = 10;
216 static unsigned long alloc_count;
217 static unsigned long branch_count;
218 static unsigned long branch_load_count;
219 -static unsigned long remap_count;
219 static unsigned long object_count;
220 static unsigned long duplicate_count;
221 static unsigned long marks_set_count;
@@ -235,6 +234,10 @@ static unsigned int atom_cnt;
234 static struct atom_str **atom_table;
235
236 /* The .pack file being generated */
237 +static const char *base_name;
238 +static unsigned int pack_count;
239 +static char *pack_name;
240 +static char *idx_name;
241 static struct packed_git *pack_data;
242 static int pack_fd;
243 static unsigned long pack_size;
@@ -593,6 +596,124 @@ static void yread(int fd, void *buffer, size_t length)
596 }
597 }
598
599 +static void start_packfile()
600 +{
601 + struct pack_header hdr;
602 +
603 + pack_count++;
604 + pack_name = xmalloc(strlen(base_name) + 11);
605 + idx_name = xmalloc(strlen(base_name) + 11);
606 + sprintf(pack_name, "%s%5.5i.pack", base_name, pack_count);
607 + sprintf(idx_name, "%s%5.5i.idx", base_name, pack_count);
608 +
609 + pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
610 + if (pack_fd < 0)
611 + die("Can't create %s: %s", pack_name, strerror(errno));
612 +
613 + pack_data = xcalloc(1, sizeof(*pack_data) + strlen(pack_name) + 2);
614 + strcpy(pack_data->pack_name, pack_name);
615 + pack_data->pack_fd = pack_fd;
616 +
617 + hdr.hdr_signature = htonl(PACK_SIGNATURE);
618 + hdr.hdr_version = htonl(2);
619 + hdr.hdr_entries = 0;
620 +
621 + write_or_die(pack_fd, &hdr, sizeof(hdr));
622 + pack_size = sizeof(hdr);
623 + object_count = 0;
624 +}
625 +
626 +static void fixup_header_footer()
627 +{
628 + SHA_CTX c;
629 + char hdr[8];
630 + unsigned long cnt;
631 + char *buf;
632 +
633 + if (lseek(pack_fd, 0, SEEK_SET) != 0)
634 + die("Failed seeking to start: %s", strerror(errno));
635 +
636 + SHA1_Init(&c);
637 + yread(pack_fd, hdr, 8);
638 + SHA1_Update(&c, hdr, 8);
639 +
640 + cnt = htonl(object_count);
641 + SHA1_Update(&c, &cnt, 4);
642 + write_or_die(pack_fd, &cnt, 4);
643 +
644 + buf = xmalloc(128 * 1024);
645 + for (;;) {
646 + size_t n = xread(pack_fd, buf, 128 * 1024);
647 + if (n <= 0)
648 + break;
649 + SHA1_Update(&c, buf, n);
650 + }
651 + free(buf);
652 +
653 + SHA1_Final(pack_sha1, &c);
654 + write_or_die(pack_fd, pack_sha1, sizeof(pack_sha1));
655 +}
656 +
657 +static int oecmp (const void *a_, const void *b_)
658 +{
659 + struct object_entry *a = *((struct object_entry**)a_);
660 + struct object_entry *b = *((struct object_entry**)b_);
661 + return hashcmp(a->sha1, b->sha1);
662 +}
663 +
664 +static void write_index(const char *idx_name)
665 +{
666 + struct sha1file *f;
667 + struct object_entry **idx, **c, **last, *e;
668 + struct object_entry_pool *o;
669 + unsigned int array[256];
670 + int i;
671 +
672 + /* Build the sorted table of object IDs. */
673 + idx = xmalloc(object_count * sizeof(struct object_entry*));
674 + c = idx;
675 + for (o = blocks; o; o = o->next_pool)
676 + for (e = o->entries; e != o->next_free; e++)
677 + *c++ = e;
678 + last = idx + object_count;
679 + qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
680 +
681 + /* Generate the fan-out array. */
682 + c = idx;
683 + for (i = 0; i < 256; i++) {
684 + struct object_entry **next = c;;
685 + while (next < last) {
686 + if ((*next)->sha1[0] != i)
687 + break;
688 + next++;
689 + }
690 + array[i] = htonl(next - idx);
691 + c = next;
692 + }
693 +
694 + f = sha1create("%s", idx_name);
695 + sha1write(f, array, 256 * sizeof(int));
696 + for (c = idx; c != last; c++) {
697 + unsigned int offset = htonl((*c)->offset);
698 + sha1write(f, &offset, 4);
699 + sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
700 + }
701 + sha1write(f, pack_sha1, sizeof(pack_sha1));
702 + sha1close(f, NULL, 1);
703 + free(idx);
704 +}
705 +
706 +static void end_packfile()
707 +{
708 + fixup_header_footer();
709 + close(pack_fd);
710 + write_index(idx_name);
711 +
712 + free(pack_name);
713 + free(idx_name);
714 + free(pack_data);
715 +}
716 +
717 static size_t encode_header(
718 enum object_type type,
719 size_t size,
@@ -994,100 +1115,6 @@ del_entry:
1115 return 1;
1116 }
1117
997 -static void init_pack_header()
998 -{
999 - struct pack_header hdr;
1000 -
1001 - hdr.hdr_signature = htonl(PACK_SIGNATURE);
1002 - hdr.hdr_version = htonl(2);
1003 - hdr.hdr_entries = 0;
1004 -
1005 - write_or_die(pack_fd, &hdr, sizeof(hdr));
1006 - pack_size = sizeof(hdr);
1007 -}
1008 -
1009 -static void fixup_header_footer()
1010 -{
1011 - SHA_CTX c;
1012 - char hdr[8];
1013 - unsigned long cnt;
1014 - char *buf;
1015 - size_t n;
1016 -
1017 - if (lseek(pack_fd, 0, SEEK_SET) != 0)
1018 - die("Failed seeking to start: %s", strerror(errno));
1019 -
1020 - SHA1_Init(&c);
1021 - yread(pack_fd, hdr, 8);
1022 - SHA1_Update(&c, hdr, 8);
1023 -
1024 - cnt = htonl(object_count);
1025 - SHA1_Update(&c, &cnt, 4);
1026 - write_or_die(pack_fd, &cnt, 4);
1027 -
1028 - buf = xmalloc(128 * 1024);
1029 - for (;;) {
1030 - n = xread(pack_fd, buf, 128 * 1024);
1031 - if (n <= 0)
1032 - break;
1033 - SHA1_Update(&c, buf, n);
1034 - }
1035 - free(buf);
1036 -
1037 - SHA1_Final(pack_sha1, &c);
1038 - write_or_die(pack_fd, pack_sha1, sizeof(pack_sha1));
1039 -}
1040 -
1041 -static int oecmp (const void *_a, const void *_b)
1042 -{
1043 - struct object_entry *a = *((struct object_entry**)_a);
1044 - struct object_entry *b = *((struct object_entry**)_b);
1045 - return hashcmp(a->sha1, b->sha1);
1046 -}
1047 -
1048 -static void write_index(const char *idx_name)
1049 -{
1050 - struct sha1file *f;
1051 - struct object_entry **idx, **c, **last;
1052 - struct object_entry *e;
1053 - struct object_entry_pool *o;
1054 - unsigned int array[256];
1055 - int i;
1056 -
1057 - /* Build the sorted table of object IDs. */
1058 - idx = xmalloc(object_count * sizeof(struct object_entry*));
1059 - c = idx;
1060 - for (o = blocks; o; o = o->next_pool)
1061 - for (e = o->entries; e != o->next_free; e++)
1062 - *c++ = e;
1063 - last = idx + object_count;
1064 - qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
1065 -
1066 - /* Generate the fan-out array. */
1067 - c = idx;
1068 - for (i = 0; i < 256; i++) {
1069 - struct object_entry **next = c;;
1070 - while (next < last) {
1071 - if ((*next)->sha1[0] != i)
1072 - break;
1073 - next++;
1074 - }
1075 - array[i] = htonl(next - idx);
1076 - c = next;
1077 - }
1078 -
1079 - f = sha1create("%s", idx_name);
1080 - sha1write(f, array, 256 * sizeof(int));
1081 - for (c = idx; c != last; c++) {
1082 - unsigned int offset = htonl((*c)->offset);
1083 - sha1write(f, &offset, 4);
1084 - sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
1085 - }
1086 - sha1write(f, pack_sha1, sizeof(pack_sha1));
1087 - sha1close(f, NULL, 1);
1088 - free(idx);
1089 -}
1090 -
1118 static void dump_branches()
1119 {
1120 static const char *msg = "fast-import";
@@ -1693,11 +1720,8 @@ static const char fast_import_usage[] =
1720
1721 int main(int argc, const char **argv)
1722 {
1696 - const char *base_name;
1723 int i;
1724 unsigned long est_obj_cnt = object_entry_alloc;
1699 - char *pack_name;
1700 - char *idx_name;
1725 struct stat sb;
1726
1727 setup_ident();
@@ -1728,20 +1752,6 @@ int main(int argc, const char **argv)
1752 usage(fast_import_usage);
1753 base_name = argv[i];
1754
1731 - pack_name = xmalloc(strlen(base_name) + 6);
1732 - sprintf(pack_name, "%s.pack", base_name);
1733 - idx_name = xmalloc(strlen(base_name) + 5);
1734 - sprintf(idx_name, "%s.idx", base_name);
1735 -
1736 - pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
1737 - if (pack_fd < 0)
1738 - die("Can't create %s: %s", pack_name, strerror(errno));
1739 -
1740 - pack_data = xcalloc(1, sizeof(*pack_data) + strlen(pack_name) + 2);
1741 - strcpy(pack_data->pack_name, pack_name);
1742 - pack_data->pack_fd = pack_fd;
1743 -
1744 - init_pack_header();
1755 alloc_objects(est_obj_cnt);
1756 strbuf_init(&command_buf);
1757
@@ -1750,6 +1760,7 @@ int main(int argc, const char **argv)
1760 avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
1761 marks = pool_calloc(1, sizeof(struct mark_set));
1762
1763 + start_packfile();
1764 for (;;) {
1765 read_next_command();
1766 if (command_buf.eof)
@@ -1765,10 +1776,8 @@ int main(int argc, const char **argv)
1776 else
1777 die("Unsupported command: %s", command_buf.buf);
1778 }
1779 + end_packfile();
1780
1769 - fixup_header_footer();
1770 - close(pack_fd);
1771 - write_index(idx_name);
1781 dump_branches();
1782 dump_tags();
1783 dump_marks();
@@ -1789,13 +1798,7 @@ int main(int argc, const char **argv)
1798 fprintf(stderr, "Memory total: %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1799 fprintf(stderr, " pools: %10lu KiB\n", total_allocd/1024);
1800 fprintf(stderr, " objects: %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1792 - fprintf(stderr, "Pack remaps: %10lu\n", remap_count);
1793 - stat(pack_name, &sb);
1794 - fprintf(stderr, "Pack size: %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1795 - stat(idx_name, &sb);
1796 - fprintf(stderr, "Index size: %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1801 fprintf(stderr, "---------------------------------------------------------------------\n");
1798 -
1802 fprintf(stderr, "\n");
1803
1804 return 0;