@matheusbsbs / matheus-bagliano / commits / ebe27b137c
Fork 0

git-repack --max-pack-size: split packs as asked by write_{object,one}()

Rewrite write_pack_file() to break to a new packfile whenever write_object/write_one request it, and correct the header's object count in the previous packfile. Change write_index_file() to write an index for just the objects in the most recent packfile. Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>

Dana L. How committed May 13, 2007 at 12:09 UTC ebe27b137c971d6d436c8d2125296ee526f6d02b
1 file changed +71 -49
builtin-pack-objects.c
+71 -49
@@ -568,6 +568,7 @@ static off_t write_one(struct sha1file *f,
568 e->offset = 0;
569 return 0;
570 }
571 + written_list[nr_written++] = e;
572
573 /* make sure off_t is sufficiently large not to wrap */
574 if (offset > offset + size)
@@ -587,47 +588,57 @@ static int adjust_perm(const char *path, mode_t mode);
588
589 static void write_pack_file(void)
590 {
590 - uint32_t i;
591 + uint32_t i = 0, j;
592 struct sha1file *f;
592 - off_t offset, last_obj_offset = 0;
593 + off_t offset, offset_one, last_obj_offset = 0;
594 struct pack_header hdr;
594 - int do_progress = progress;
595 -
596 - if (pack_to_stdout) {
597 - f = sha1fd(1, "<stdout>");
598 - do_progress >>= 1;
599 - } else {
600 - int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
601 - if (fd < 0)
602 - die("unable to create %s: %s\n", tmpname, strerror(errno));
603 - pack_tmp_name = xstrdup(tmpname);
604 - f = sha1fd(fd, pack_tmp_name);
605 - }
595 + int do_progress = progress >> pack_to_stdout;
596 + uint32_t nr_remaining = nr_result;
597
598 if (do_progress)
599 start_progress(&progress_state, "Writing %u objects...", "", nr_result);
600 + written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
601
610 - hdr.hdr_signature = htonl(PACK_SIGNATURE);
611 - hdr.hdr_version = htonl(PACK_VERSION);
612 - hdr.hdr_entries = htonl(nr_result);
613 - sha1write(f, &hdr, sizeof(hdr));
614 - offset = sizeof(hdr);
615 - if (!nr_result)
616 - goto done;
617 - for (i = 0; i < nr_objects; i++) {
618 - last_obj_offset = offset;
619 - offset = write_one(f, objects + i, offset);
620 - if (do_progress)
621 - display_progress(&progress_state, written);
622 - }
623 - if (do_progress)
624 - stop_progress(&progress_state);
625 - done:
626 - if (written != nr_result)
627 - die("wrote %u objects while expecting %u", written, nr_result);
628 - sha1close(f, pack_file_sha1, 1);
602 + do {
603 + if (pack_to_stdout) {
604 + f = sha1fd(1, "<stdout>");
605 + } else {
606 + int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
607 + if (fd < 0)
608 + die("unable to create %s: %s\n", tmpname, strerror(errno));
609 + pack_tmp_name = xstrdup(tmpname);
610 + f = sha1fd(fd, pack_tmp_name);
611 + }
612
630 - if (!pack_to_stdout) {
613 + hdr.hdr_signature = htonl(PACK_SIGNATURE);
614 + hdr.hdr_version = htonl(PACK_VERSION);
615 + hdr.hdr_entries = htonl(nr_remaining);
616 + sha1write(f, &hdr, sizeof(hdr));
617 + offset = sizeof(hdr);
618 + nr_written = 0;
619 + for (; i < nr_objects; i++) {
620 + last_obj_offset = offset;
621 + offset_one = write_one(f, objects + i, offset);
622 + if (!offset_one)
623 + break;
624 + offset = offset_one;
625 + if (do_progress)
626 + display_progress(&progress_state, written);
627 + }
628 +
629 + /*
630 + * Did we write the wrong # entries in the header?
631 + * If so, rewrite it like in fast-import
632 + */
633 + if (pack_to_stdout || nr_written == nr_remaining) {
634 + sha1close(f, pack_file_sha1, 1);
635 + } else {
636 + sha1close(f, pack_file_sha1, 0);
637 + fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
638 + close(f->fd);
639 + }
640 +
641 + if (!pack_to_stdout) {
642 unsigned char object_list_sha1[20];
643 mode_t mode = umask(0);
644
@@ -652,7 +663,26 @@ static void write_pack_file(void)
663 die("unable to rename temporary index file: %s",
664 strerror(errno));
665 puts(sha1_to_hex(object_list_sha1));
666 + }
667 +
668 + /* mark written objects as written to previous pack */
669 + for (j = 0; j < nr_written; j++) {
670 + written_list[j]->offset = (off_t)-1;
671 + }
672 + nr_remaining -= nr_written;
673 + } while (nr_remaining && i < nr_objects);
674 +
675 + free(written_list);
676 + if (do_progress)
677 + stop_progress(&progress_state);
678 + if (written != nr_result)
679 + die("wrote %u objects while expecting %u", written, nr_result);
680 + for (j = 0; i < nr_objects; i++) {
681 + struct object_entry *e = objects + i;
682 + j += !e->offset && !e->preferred_base;
683 }
684 + if (j)
685 + die("wrote %u objects as expected but %u unwritten", written, j);
686 }
687
688 static int sha1_sort(const void *_a, const void *_b)
@@ -679,18 +709,11 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
709 idx_tmp_name = xstrdup(tmpname);
710 f = sha1fd(fd, idx_tmp_name);
711
682 - if (nr_result) {
683 - uint32_t j = 0;
684 - sorted_by_sha =
685 - xcalloc(nr_result, sizeof(struct object_entry *));
686 - for (i = 0; i < nr_objects; i++)
687 - if (!objects[i].preferred_base)
688 - sorted_by_sha[j++] = objects + i;
689 - if (j != nr_result)
690 - die("listed %u objects while expecting %u", j, nr_result);
691 - qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
712 + if (nr_written) {
713 + sorted_by_sha = written_list;
714 + qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
715 list = sorted_by_sha;
693 - last = sorted_by_sha + nr_result;
716 + last = sorted_by_sha + nr_written;
717 } else
718 sorted_by_sha = list = last = NULL;
719
@@ -728,7 +751,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
751
752 /* Write the actual SHA1 entries. */
753 list = sorted_by_sha;
731 - for (i = 0; i < nr_result; i++) {
754 + for (i = 0; i < nr_written; i++) {
755 struct object_entry *entry = *list++;
756 if (index_version < 2) {
757 uint32_t offset = htonl(entry->offset);
@@ -743,7 +766,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
766
767 /* write the crc32 table */
768 list = sorted_by_sha;
746 - for (i = 0; i < nr_objects; i++) {
769 + for (i = 0; i < nr_written; i++) {
770 struct object_entry *entry = *list++;
771 uint32_t crc32_val = htonl(entry->crc32);
772 sha1write(f, &crc32_val, 4);
@@ -751,7 +774,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
774
775 /* write the 32-bit offset table */
776 list = sorted_by_sha;
754 - for (i = 0; i < nr_objects; i++) {
777 + for (i = 0; i < nr_written; i++) {
778 struct object_entry *entry = *list++;
779 uint32_t offset = (entry->offset <= index_off32_limit) ?
780 entry->offset : (0x80000000 | nr_large_offset++);
@@ -776,7 +799,6 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
799
800 sha1write(f, pack_file_sha1, 20);
801 sha1close(f, NULL, 1);
779 - free(sorted_by_sha);
802 SHA1_Final(sha1, &ctx);
803 }
804