@matheusbsbs / matheus-bagliano / commits / d489bc1491
Fork 0

Improve reuse of sha1_file library within fast-import.

Now that the sha1_file.c library routines use the sliding mmap routines to perform efficient access to portions of a packfile I can remove that code from fast-import.c and just invoke it. One benefit is we now have reloading support for any packfile which uses OBJ_OFS_DELTA. Another is we have significantly less code to maintain. This code reuse change *requires* that fast-import generate only an OBJ_OFS_DELTA format packfile, as there is absolutely no index available to perform OBJ_REF_DELTA lookup in while unpacking an object. This is probably reasonable to require as the delta offsets result in smaller packfiles and are faster to unpack, as no index searching is required. Its also only a temporary requirement as users could always repack without offsets before making the import available to older versions of Git. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Jan 14, 2007 at 06:20 UTC d489bc14919cdd37d3978065591199d21d6719f8
1 file changed +31 -144
fast-import.c
+31 -144
@@ -136,9 +136,9 @@ struct last_object
136 {
137 void *data;
138 unsigned long len;
139 + unsigned long offset;
140 unsigned int depth;
140 - int no_free;
141 - unsigned char sha1[20];
141 + unsigned no_free:1;
142 };
143
144 struct mem_pool
@@ -235,13 +235,10 @@ static unsigned int atom_cnt;
235 static struct atom_str **atom_table;
236
237 /* The .pack file being generated */
238 +static struct packed_git *pack_data;
239 static int pack_fd;
240 static unsigned long pack_size;
241 static unsigned char pack_sha1[20];
241 -static unsigned char* pack_base;
242 -static unsigned long pack_moff;
243 -static unsigned long pack_mlen = 128*1024*1024;
244 -static unsigned long page_size;
242
243 /* Table of objects we've written. */
244 static unsigned int object_entry_alloc = 5000;
@@ -667,14 +664,23 @@ static int store_object(
664 deflateInit(&s, zlib_compression_level);
665
666 if (delta) {
667 + unsigned long ofs = e->offset - last->offset;
668 + unsigned pos = sizeof(hdr) - 1;
669 +
670 delta_count_by_type[type]++;
671 last->depth++;
672 s.next_in = delta;
673 s.avail_in = deltalen;
674 - hdrlen = encode_header(OBJ_REF_DELTA, deltalen, hdr);
674 +
675 + hdrlen = encode_header(OBJ_OFS_DELTA, deltalen, hdr);
676 write_or_die(pack_fd, hdr, hdrlen);
676 - write_or_die(pack_fd, last->sha1, sizeof(sha1));
677 - pack_size += hdrlen + sizeof(sha1);
677 + pack_size += hdrlen;
678 +
679 + hdr[pos] = ofs & 127;
680 + while (ofs >>= 7)
681 + hdr[--pos] = 128 | (--ofs & 127);
682 + write_or_die(pack_fd, hdr + pos, sizeof(hdr) - pos);
683 + pack_size += sizeof(hdr) - pos;
684 } else {
685 if (last)
686 last->depth = 0;
@@ -701,139 +707,17 @@ static int store_object(
707 if (last->data && !last->no_free)
708 free(last->data);
709 last->data = dat;
710 + last->offset = e->offset;
711 last->len = datlen;
705 - hashcpy(last->sha1, sha1);
712 }
713 return 0;
714 }
715
710 -static unsigned char* map_pack(unsigned long offset, unsigned int *left)
711 -{
712 - if (offset >= pack_size)
713 - die("object offset outside of pack file");
714 - if (!pack_base
715 - || offset < pack_moff
716 - || (offset + 20) >= (pack_moff + pack_mlen)) {
717 - if (pack_base)
718 - munmap(pack_base, pack_mlen);
719 - pack_moff = (offset / page_size) * page_size;
720 - pack_base = mmap(NULL,pack_mlen,PROT_READ,MAP_SHARED,
721 - pack_fd,pack_moff);
722 - if (pack_base == MAP_FAILED)
723 - die("Failed to map generated pack: %s", strerror(errno));
724 - remap_count++;
725 - }
726 - offset -= pack_moff;
727 - if (left)
728 - *left = pack_mlen - offset;
729 - return pack_base + offset;
730 -}
731 -
732 -static unsigned long unpack_object_header(unsigned long offset,
733 - enum object_type *type,
734 - unsigned long *sizep)
735 -{
736 - unsigned shift;
737 - unsigned char c;
738 - unsigned long size;
739 -
740 - c = *map_pack(offset++, NULL);
741 - *type = (c >> 4) & 7;
742 - size = c & 15;
743 - shift = 4;
744 - while (c & 0x80) {
745 - c = *map_pack(offset++, NULL);
746 - size += (c & 0x7f) << shift;
747 - shift += 7;
748 - }
749 - *sizep = size;
750 - return offset;
751 -}
752 -
753 -static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
754 -{
755 - z_stream stream;
756 - unsigned char *result;
757 -
758 - result = xmalloc(sz + 1);
759 - result[sz] = 0;
760 -
761 - memset(&stream, 0, sizeof(stream));
762 - stream.next_in = map_pack(o, &stream.avail_in);
763 - stream.next_out = result;
764 - stream.avail_out = sz;
765 -
766 - inflateInit(&stream);
767 - for (;;) {
768 - int st = inflate(&stream, Z_FINISH);
769 - if (st == Z_STREAM_END)
770 - break;
771 - if (st == Z_OK || st == Z_BUF_ERROR) {
772 - o = stream.next_in - pack_base + pack_moff;
773 - stream.next_in = map_pack(o, &stream.avail_in);
774 - continue;
775 - }
776 - die("Error %i from zlib during inflate.", st);
777 - }
778 - inflateEnd(&stream);
779 - if (stream.total_out != sz)
780 - die("Error after inflate: sizes mismatch");
781 - return result;
782 -}
783 -
784 -static void *gfi_unpack_entry(unsigned long offset,
785 - unsigned long *sizep,
786 - unsigned int *delta_depth);
787 -
788 -static void *unpack_delta_entry(unsigned long offset,
789 - unsigned long delta_size,
790 - unsigned long *sizep,
791 - unsigned int *delta_depth)
792 -{
793 - struct object_entry *base_oe;
794 - unsigned char *base_sha1;
795 - void *delta_data, *base, *result;
796 - unsigned long base_size, result_size;
797 -
798 - base_sha1 = map_pack(offset, NULL);
799 - base_oe = find_object(base_sha1);
800 - if (!base_oe)
801 - die("I'm broken; I can't find a base I know must be here.");
802 - base = gfi_unpack_entry(base_oe->offset, &base_size, delta_depth);
803 - delta_data = unpack_non_delta_entry(offset + 20, delta_size);
804 - result = patch_delta(base, base_size,
805 - delta_data, delta_size,
806 - &result_size);
807 - if (!result)
808 - die("failed to apply delta");
809 - free(delta_data);
810 - free(base);
811 - *sizep = result_size;
812 - (*delta_depth)++;
813 - return result;
814 -}
815 -
816 -static void *gfi_unpack_entry(unsigned long offset,
817 - unsigned long *sizep,
818 - unsigned int *delta_depth)
716 +static void *gfi_unpack_entry(unsigned long ofs, unsigned long *sizep)
717 {
820 - unsigned long size;
821 - enum object_type kind;
822 -
823 - offset = unpack_object_header(offset, &kind, &size);
824 - switch (kind) {
825 - case OBJ_REF_DELTA:
826 - return unpack_delta_entry(offset, size, sizep, delta_depth);
827 - case OBJ_COMMIT:
828 - case OBJ_TREE:
829 - case OBJ_BLOB:
830 - case OBJ_TAG:
831 - *sizep = size;
832 - *delta_depth = 0;
833 - return unpack_non_delta_entry(offset, size);
834 - default:
835 - die("I created an object I can't read!");
836 - }
718 + char type[20];
719 + pack_data->pack_size = pack_size + 20;
720 + return unpack_entry(pack_data, ofs, type, sizep);
721 }
722
723 static const char *get_mode(const char *str, unsigned int *modep)
@@ -867,7 +751,8 @@ static void load_tree(struct tree_entry *root)
751 if (myoe) {
752 if (myoe->type != OBJ_TREE)
753 die("Not a tree: %s", sha1_to_hex(sha1));
870 - buf = gfi_unpack_entry(myoe->offset, &size, &t->delta_depth);
754 + t->delta_depth = 0;
755 + buf = gfi_unpack_entry(myoe->offset, &size);
756 } else {
757 char type[20];
758 buf = read_sha1_file(sha1, type, &size);
@@ -956,6 +841,7 @@ static void store_tree(struct tree_entry *root)
841 unsigned int i, j, del;
842 unsigned long new_len;
843 struct last_object lo;
844 + struct object_entry *le;
845
846 if (!is_null_sha1(root->versions[1].sha1))
847 return;
@@ -965,17 +851,16 @@ static void store_tree(struct tree_entry *root)
851 store_tree(t->entries[i]);
852 }
853
968 - if (!S_ISDIR(root->versions[0].mode)
969 - || is_null_sha1(root->versions[0].sha1)
970 - || !find_object(root->versions[0].sha1)) {
854 + le = find_object(root->versions[0].sha1);
855 + if (!S_ISDIR(root->versions[0].mode) || !le) {
856 lo.data = NULL;
857 lo.depth = 0;
858 } else {
859 mktree(t, 0, &lo.len, &old_tree);
860 lo.data = old_tree.buffer;
861 + lo.offset = le->offset;
862 lo.depth = t->delta_depth;
863 lo.no_free = 1;
978 - hashcpy(lo.sha1, root->versions[0].sha1);
864 }
865
866 mktree(t, 1, &new_len, &new_tree);
@@ -1471,12 +1356,11 @@ static void cmd_from(struct branch *b)
1356 unsigned long idnum = strtoul(from + 1, NULL, 10);
1357 struct object_entry *oe = find_mark(idnum);
1358 unsigned long size;
1474 - unsigned int depth;
1359 char *buf;
1360 if (oe->type != OBJ_COMMIT)
1361 die("Mark :%lu not a commit", idnum);
1362 hashcpy(b->sha1, oe->sha1);
1479 - buf = gfi_unpack_entry(oe->offset, &size, &depth);
1363 + buf = gfi_unpack_entry(oe->offset, &size);
1364 if (!buf || size < 46)
1365 die("Not a valid commit: %s", from);
1366 if (memcmp("tree ", buf, 5)
@@ -1818,7 +1702,6 @@ int main(int argc, const char **argv)
1702
1703 setup_ident();
1704 git_config(git_default_config);
1821 - page_size = getpagesize();
1705
1706 for (i = 1; i < argc; i++) {
1707 const char *a = argv[i];
@@ -1854,6 +1737,10 @@ int main(int argc, const char **argv)
1737 if (pack_fd < 0)
1738 die("Can't create %s: %s", pack_name, strerror(errno));
1739
1740 + pack_data = xcalloc(1, sizeof(*pack_data) + strlen(pack_name) + 2);
1741 + strcpy(pack_data->pack_name, pack_name);
1742 + pack_data->pack_fd = pack_fd;
1743 +
1744 init_pack_header();
1745 alloc_objects(est_obj_cnt);
1746 strbuf_init(&command_buf);