git-repack --max-pack-size: split packs as asked by write_{object,one}()
Rewrite write_pack_file() to break to a new packfile whenever write_object/write_one request it, and correct the header's object count in the previous packfile. Change write_index_file() to write an index for just the objects in the most recent packfile. Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>
Dana L. How committed
May 13, 2007 at 12:09 UTC
ebe27b137c971d6d436c8d2125296ee526f6d02b
1 file changed
+71
-49
builtin-pack-objects.c
+71
-49
@@ -568,6 +568,7 @@ static off_t write_one(struct sha1file *f,
568
e->offset = 0;
569
return 0;
570
}
571
+ written_list[nr_written++] = e;
572
573
/* make sure off_t is sufficiently large not to wrap */
574
if (offset > offset + size)
@@ -587,47 +588,57 @@ static int adjust_perm(const char *path, mode_t mode);
588
589
static void write_pack_file(void)
590
{
590
- uint32_t i;
591
+ uint32_t i = 0, j;
592
struct sha1file *f;
592
- off_t offset, last_obj_offset = 0;
593
+ off_t offset, offset_one, last_obj_offset = 0;
594
struct pack_header hdr;
594
- int do_progress = progress;
595
-
596
- if (pack_to_stdout) {
597
- f = sha1fd(1, "<stdout>");
598
- do_progress >>= 1;
599
- } else {
600
- int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
601
- if (fd < 0)
602
- die("unable to create %s: %s\n", tmpname, strerror(errno));
603
- pack_tmp_name = xstrdup(tmpname);
604
- f = sha1fd(fd, pack_tmp_name);
605
- }
595
+ int do_progress = progress >> pack_to_stdout;
596
+ uint32_t nr_remaining = nr_result;
597
598
if (do_progress)
599
start_progress(&progress_state, "Writing %u objects...", "", nr_result);
600
+ written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
601
610
- hdr.hdr_signature = htonl(PACK_SIGNATURE);
611
- hdr.hdr_version = htonl(PACK_VERSION);
612
- hdr.hdr_entries = htonl(nr_result);
613
- sha1write(f, &hdr, sizeof(hdr));
614
- offset = sizeof(hdr);
615
- if (!nr_result)
616
- goto done;
617
- for (i = 0; i < nr_objects; i++) {
618
- last_obj_offset = offset;
619
- offset = write_one(f, objects + i, offset);
620
- if (do_progress)
621
- display_progress(&progress_state, written);
622
- }
623
- if (do_progress)
624
- stop_progress(&progress_state);
625
- done:
626
- if (written != nr_result)
627
- die("wrote %u objects while expecting %u", written, nr_result);
628
- sha1close(f, pack_file_sha1, 1);
602
+ do {
603
+ if (pack_to_stdout) {
604
+ f = sha1fd(1, "<stdout>");
605
+ } else {
606
+ int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
607
+ if (fd < 0)
608
+ die("unable to create %s: %s\n", tmpname, strerror(errno));
609
+ pack_tmp_name = xstrdup(tmpname);
610
+ f = sha1fd(fd, pack_tmp_name);
611
+ }
612
630
- if (!pack_to_stdout) {
613
+ hdr.hdr_signature = htonl(PACK_SIGNATURE);
614
+ hdr.hdr_version = htonl(PACK_VERSION);
615
+ hdr.hdr_entries = htonl(nr_remaining);
616
+ sha1write(f, &hdr, sizeof(hdr));
617
+ offset = sizeof(hdr);
618
+ nr_written = 0;
619
+ for (; i < nr_objects; i++) {
620
+ last_obj_offset = offset;
621
+ offset_one = write_one(f, objects + i, offset);
622
+ if (!offset_one)
623
+ break;
624
+ offset = offset_one;
625
+ if (do_progress)
626
+ display_progress(&progress_state, written);
627
+ }
628
+
629
+ /*
630
+ * Did we write the wrong # entries in the header?
631
+ * If so, rewrite it like in fast-import
632
+ */
633
+ if (pack_to_stdout || nr_written == nr_remaining) {
634
+ sha1close(f, pack_file_sha1, 1);
635
+ } else {
636
+ sha1close(f, pack_file_sha1, 0);
637
+ fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
638
+ close(f->fd);
639
+ }
640
+
641
+ if (!pack_to_stdout) {
642
unsigned char object_list_sha1[20];
643
mode_t mode = umask(0);
644
@@ -652,7 +663,26 @@ static void write_pack_file(void)
663
die("unable to rename temporary index file: %s",
664
strerror(errno));
665
puts(sha1_to_hex(object_list_sha1));
666
+ }
667
+
668
+ /* mark written objects as written to previous pack */
669
+ for (j = 0; j < nr_written; j++) {
670
+ written_list[j]->offset = (off_t)-1;
671
+ }
672
+ nr_remaining -= nr_written;
673
+ } while (nr_remaining && i < nr_objects);
674
+
675
+ free(written_list);
676
+ if (do_progress)
677
+ stop_progress(&progress_state);
678
+ if (written != nr_result)
679
+ die("wrote %u objects while expecting %u", written, nr_result);
680
+ for (j = 0; i < nr_objects; i++) {
681
+ struct object_entry *e = objects + i;
682
+ j += !e->offset && !e->preferred_base;
683
}
684
+ if (j)
685
+ die("wrote %u objects as expected but %u unwritten", written, j);
686
}
687
688
static int sha1_sort(const void *_a, const void *_b)
@@ -679,18 +709,11 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
709
idx_tmp_name = xstrdup(tmpname);
710
f = sha1fd(fd, idx_tmp_name);
711
682
- if (nr_result) {
683
- uint32_t j = 0;
684
- sorted_by_sha =
685
- xcalloc(nr_result, sizeof(struct object_entry *));
686
- for (i = 0; i < nr_objects; i++)
687
- if (!objects[i].preferred_base)
688
- sorted_by_sha[j++] = objects + i;
689
- if (j != nr_result)
690
- die("listed %u objects while expecting %u", j, nr_result);
691
- qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
712
+ if (nr_written) {
713
+ sorted_by_sha = written_list;
714
+ qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
715
list = sorted_by_sha;
693
- last = sorted_by_sha + nr_result;
716
+ last = sorted_by_sha + nr_written;
717
} else
718
sorted_by_sha = list = last = NULL;
719
@@ -728,7 +751,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
751
752
/* Write the actual SHA1 entries. */
753
list = sorted_by_sha;
731
- for (i = 0; i < nr_result; i++) {
754
+ for (i = 0; i < nr_written; i++) {
755
struct object_entry *entry = *list++;
756
if (index_version < 2) {
757
uint32_t offset = htonl(entry->offset);
@@ -743,7 +766,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
766
767
/* write the crc32 table */
768
list = sorted_by_sha;
746
- for (i = 0; i < nr_objects; i++) {
769
+ for (i = 0; i < nr_written; i++) {
770
struct object_entry *entry = *list++;
771
uint32_t crc32_val = htonl(entry->crc32);
772
sha1write(f, &crc32_val, 4);
@@ -751,7 +774,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
774
775
/* write the 32-bit offset table */
776
list = sorted_by_sha;
754
- for (i = 0; i < nr_objects; i++) {
777
+ for (i = 0; i < nr_written; i++) {
778
struct object_entry *entry = *list++;
779
uint32_t offset = (entry->offset <= index_off32_limit) ?
780
entry->offset : (0x80000000 | nr_large_offset++);
@@ -776,7 +799,6 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
799
800
sha1write(f, pack_file_sha1, 20);
801
sha1close(f, NULL, 1);
779
- free(sorted_by_sha);
802
SHA1_Final(sha1, &ctx);
803
}
804