Restructure fast-import to support creating multiple packfiles.
Now that we are starting to see some really large projects (such as KDE or a fork of FreeBSD) get imported into Git we're running into the upper limit on packfile object count as well as overall byte length. The KDE and FreeBSD projects are both likely to require more than 4 GiB to store their current history, which means we really need multiple packfiles to handle their content. This is a fairly simple restructuring of the internal code to help us support creating multiple packfiles from within fast-import. We are now adding a 5 digit incrementing suffix to the end of the basename supplied to us by the caller, permitting up to 99,999 packs to be generated in a single fast-import run. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Jan 15, 2007 at 04:39 UTC
f70b653429ebc7fdde0b36a63e1deb4aadb450d3
1 file changed
+124
-121
fast-import.c
+124
-121
@@ -216,7 +216,6 @@ static unsigned long max_depth = 10;
216
static unsigned long alloc_count;
217
static unsigned long branch_count;
218
static unsigned long branch_load_count;
219
-static unsigned long remap_count;
219
static unsigned long object_count;
220
static unsigned long duplicate_count;
221
static unsigned long marks_set_count;
@@ -235,6 +234,10 @@ static unsigned int atom_cnt;
234
static struct atom_str **atom_table;
235
236
/* The .pack file being generated */
237
+static const char *base_name;
238
+static unsigned int pack_count;
239
+static char *pack_name;
240
+static char *idx_name;
241
static struct packed_git *pack_data;
242
static int pack_fd;
243
static unsigned long pack_size;
@@ -593,6 +596,124 @@ static void yread(int fd, void *buffer, size_t length)
596
}
597
}
598
599
+static void start_packfile()
600
+{
601
+ struct pack_header hdr;
602
+
603
+ pack_count++;
604
+ pack_name = xmalloc(strlen(base_name) + 11);
605
+ idx_name = xmalloc(strlen(base_name) + 11);
606
+ sprintf(pack_name, "%s%5.5i.pack", base_name, pack_count);
607
+ sprintf(idx_name, "%s%5.5i.idx", base_name, pack_count);
608
+
609
+ pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
610
+ if (pack_fd < 0)
611
+ die("Can't create %s: %s", pack_name, strerror(errno));
612
+
613
+ pack_data = xcalloc(1, sizeof(*pack_data) + strlen(pack_name) + 2);
614
+ strcpy(pack_data->pack_name, pack_name);
615
+ pack_data->pack_fd = pack_fd;
616
+
617
+ hdr.hdr_signature = htonl(PACK_SIGNATURE);
618
+ hdr.hdr_version = htonl(2);
619
+ hdr.hdr_entries = 0;
620
+
621
+ write_or_die(pack_fd, &hdr, sizeof(hdr));
622
+ pack_size = sizeof(hdr);
623
+ object_count = 0;
624
+}
625
+
626
+static void fixup_header_footer()
627
+{
628
+ SHA_CTX c;
629
+ char hdr[8];
630
+ unsigned long cnt;
631
+ char *buf;
632
+
633
+ if (lseek(pack_fd, 0, SEEK_SET) != 0)
634
+ die("Failed seeking to start: %s", strerror(errno));
635
+
636
+ SHA1_Init(&c);
637
+ yread(pack_fd, hdr, 8);
638
+ SHA1_Update(&c, hdr, 8);
639
+
640
+ cnt = htonl(object_count);
641
+ SHA1_Update(&c, &cnt, 4);
642
+ write_or_die(pack_fd, &cnt, 4);
643
+
644
+ buf = xmalloc(128 * 1024);
645
+ for (;;) {
646
+ size_t n = xread(pack_fd, buf, 128 * 1024);
647
+ if (n <= 0)
648
+ break;
649
+ SHA1_Update(&c, buf, n);
650
+ }
651
+ free(buf);
652
+
653
+ SHA1_Final(pack_sha1, &c);
654
+ write_or_die(pack_fd, pack_sha1, sizeof(pack_sha1));
655
+}
656
+
657
+static int oecmp (const void *a_, const void *b_)
658
+{
659
+ struct object_entry *a = *((struct object_entry**)a_);
660
+ struct object_entry *b = *((struct object_entry**)b_);
661
+ return hashcmp(a->sha1, b->sha1);
662
+}
663
+
664
+static void write_index(const char *idx_name)
665
+{
666
+ struct sha1file *f;
667
+ struct object_entry **idx, **c, **last, *e;
668
+ struct object_entry_pool *o;
669
+ unsigned int array[256];
670
+ int i;
671
+
672
+ /* Build the sorted table of object IDs. */
673
+ idx = xmalloc(object_count * sizeof(struct object_entry*));
674
+ c = idx;
675
+ for (o = blocks; o; o = o->next_pool)
676
+ for (e = o->entries; e != o->next_free; e++)
677
+ *c++ = e;
678
+ last = idx + object_count;
679
+ qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
680
+
681
+ /* Generate the fan-out array. */
682
+ c = idx;
683
+ for (i = 0; i < 256; i++) {
684
+ struct object_entry **next = c;;
685
+ while (next < last) {
686
+ if ((*next)->sha1[0] != i)
687
+ break;
688
+ next++;
689
+ }
690
+ array[i] = htonl(next - idx);
691
+ c = next;
692
+ }
693
+
694
+ f = sha1create("%s", idx_name);
695
+ sha1write(f, array, 256 * sizeof(int));
696
+ for (c = idx; c != last; c++) {
697
+ unsigned int offset = htonl((*c)->offset);
698
+ sha1write(f, &offset, 4);
699
+ sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
700
+ }
701
+ sha1write(f, pack_sha1, sizeof(pack_sha1));
702
+ sha1close(f, NULL, 1);
703
+ free(idx);
704
+}
705
+
706
+static void end_packfile()
707
+{
708
+ fixup_header_footer();
709
+ close(pack_fd);
710
+ write_index(idx_name);
711
+
712
+ free(pack_name);
713
+ free(idx_name);
714
+ free(pack_data);
715
+}
716
+
717
static size_t encode_header(
718
enum object_type type,
719
size_t size,
@@ -994,100 +1115,6 @@ del_entry:
1115
return 1;
1116
}
1117
997
-static void init_pack_header()
998
-{
999
- struct pack_header hdr;
1000
-
1001
- hdr.hdr_signature = htonl(PACK_SIGNATURE);
1002
- hdr.hdr_version = htonl(2);
1003
- hdr.hdr_entries = 0;
1004
-
1005
- write_or_die(pack_fd, &hdr, sizeof(hdr));
1006
- pack_size = sizeof(hdr);
1007
-}
1008
-
1009
-static void fixup_header_footer()
1010
-{
1011
- SHA_CTX c;
1012
- char hdr[8];
1013
- unsigned long cnt;
1014
- char *buf;
1015
- size_t n;
1016
-
1017
- if (lseek(pack_fd, 0, SEEK_SET) != 0)
1018
- die("Failed seeking to start: %s", strerror(errno));
1019
-
1020
- SHA1_Init(&c);
1021
- yread(pack_fd, hdr, 8);
1022
- SHA1_Update(&c, hdr, 8);
1023
-
1024
- cnt = htonl(object_count);
1025
- SHA1_Update(&c, &cnt, 4);
1026
- write_or_die(pack_fd, &cnt, 4);
1027
-
1028
- buf = xmalloc(128 * 1024);
1029
- for (;;) {
1030
- n = xread(pack_fd, buf, 128 * 1024);
1031
- if (n <= 0)
1032
- break;
1033
- SHA1_Update(&c, buf, n);
1034
- }
1035
- free(buf);
1036
-
1037
- SHA1_Final(pack_sha1, &c);
1038
- write_or_die(pack_fd, pack_sha1, sizeof(pack_sha1));
1039
-}
1040
-
1041
-static int oecmp (const void *_a, const void *_b)
1042
-{
1043
- struct object_entry *a = *((struct object_entry**)_a);
1044
- struct object_entry *b = *((struct object_entry**)_b);
1045
- return hashcmp(a->sha1, b->sha1);
1046
-}
1047
-
1048
-static void write_index(const char *idx_name)
1049
-{
1050
- struct sha1file *f;
1051
- struct object_entry **idx, **c, **last;
1052
- struct object_entry *e;
1053
- struct object_entry_pool *o;
1054
- unsigned int array[256];
1055
- int i;
1056
-
1057
- /* Build the sorted table of object IDs. */
1058
- idx = xmalloc(object_count * sizeof(struct object_entry*));
1059
- c = idx;
1060
- for (o = blocks; o; o = o->next_pool)
1061
- for (e = o->entries; e != o->next_free; e++)
1062
- *c++ = e;
1063
- last = idx + object_count;
1064
- qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
1065
-
1066
- /* Generate the fan-out array. */
1067
- c = idx;
1068
- for (i = 0; i < 256; i++) {
1069
- struct object_entry **next = c;;
1070
- while (next < last) {
1071
- if ((*next)->sha1[0] != i)
1072
- break;
1073
- next++;
1074
- }
1075
- array[i] = htonl(next - idx);
1076
- c = next;
1077
- }
1078
-
1079
- f = sha1create("%s", idx_name);
1080
- sha1write(f, array, 256 * sizeof(int));
1081
- for (c = idx; c != last; c++) {
1082
- unsigned int offset = htonl((*c)->offset);
1083
- sha1write(f, &offset, 4);
1084
- sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
1085
- }
1086
- sha1write(f, pack_sha1, sizeof(pack_sha1));
1087
- sha1close(f, NULL, 1);
1088
- free(idx);
1089
-}
1090
-
1118
static void dump_branches()
1119
{
1120
static const char *msg = "fast-import";
@@ -1693,11 +1720,8 @@ static const char fast_import_usage[] =
1720
1721
int main(int argc, const char **argv)
1722
{
1696
- const char *base_name;
1723
int i;
1724
unsigned long est_obj_cnt = object_entry_alloc;
1699
- char *pack_name;
1700
- char *idx_name;
1725
struct stat sb;
1726
1727
setup_ident();
@@ -1728,20 +1752,6 @@ int main(int argc, const char **argv)
1752
usage(fast_import_usage);
1753
base_name = argv[i];
1754
1731
- pack_name = xmalloc(strlen(base_name) + 6);
1732
- sprintf(pack_name, "%s.pack", base_name);
1733
- idx_name = xmalloc(strlen(base_name) + 5);
1734
- sprintf(idx_name, "%s.idx", base_name);
1735
-
1736
- pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
1737
- if (pack_fd < 0)
1738
- die("Can't create %s: %s", pack_name, strerror(errno));
1739
-
1740
- pack_data = xcalloc(1, sizeof(*pack_data) + strlen(pack_name) + 2);
1741
- strcpy(pack_data->pack_name, pack_name);
1742
- pack_data->pack_fd = pack_fd;
1743
-
1744
- init_pack_header();
1755
alloc_objects(est_obj_cnt);
1756
strbuf_init(&command_buf);
1757
@@ -1750,6 +1760,7 @@ int main(int argc, const char **argv)
1760
avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
1761
marks = pool_calloc(1, sizeof(struct mark_set));
1762
1763
+ start_packfile();
1764
for (;;) {
1765
read_next_command();
1766
if (command_buf.eof)
@@ -1765,10 +1776,8 @@ int main(int argc, const char **argv)
1776
else
1777
die("Unsupported command: %s", command_buf.buf);
1778
}
1779
+ end_packfile();
1780
1769
- fixup_header_footer();
1770
- close(pack_fd);
1771
- write_index(idx_name);
1781
dump_branches();
1782
dump_tags();
1783
dump_marks();
@@ -1789,13 +1798,7 @@ int main(int argc, const char **argv)
1798
fprintf(stderr, "Memory total: %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1799
fprintf(stderr, " pools: %10lu KiB\n", total_allocd/1024);
1800
fprintf(stderr, " objects: %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1792
- fprintf(stderr, "Pack remaps: %10lu\n", remap_count);
1793
- stat(pack_name, &sb);
1794
- fprintf(stderr, "Pack size: %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1795
- stat(idx_name, &sb);
1796
- fprintf(stderr, "Index size: %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1801
fprintf(stderr, "---------------------------------------------------------------------\n");
1798
-
1802
fprintf(stderr, "\n");
1803
1804
return 0;