@matheusbsbs / matheus-bagliano / commits / 41e5257fcf
Fork 0

Implemented tree reloading in fast-import.

Tree reloading allows fast-import to swap out the least-recently used branch by simply deallocating the data structures from memory that were associated with that branch. Later if the branch becomes active again it can lazily recreate those structures on demand by reloading the necessary trees from the pack file it originally wrote them to. The reloading process is implemented by mmap'ing the pack into memory and using a much tighter variant of the pack reading code contained in sha1_file.c. This was a blatent copy from sha1_file.c but the unpacking functions were significantly simplified and are actually now in a form that should make it easier to map only the necessary regions of a pack rather than the entire file. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 24, 2006 at 04:37 UTC 41e5257fcf4db31dfa2576aac1f50b140f2bb058
1 file changed +149 -13
fast-import.c
+149 -13
@@ -198,6 +198,7 @@ static unsigned long max_depth = 10;
198 static unsigned long alloc_count;
199 static unsigned long branch_count;
200 static unsigned long branch_load_count;
201 +static unsigned long remap_count;
202 static unsigned long object_count;
203 static unsigned long duplicate_count;
204 static unsigned long marks_set_count;
@@ -216,8 +217,10 @@ static struct atom_str **atom_table;
217
218 /* The .pack file being generated */
219 static int pack_fd;
219 -static unsigned long pack_offset;
220 +static unsigned long pack_size;
221 static unsigned char pack_sha1[20];
222 +static void* pack_base;
223 +static size_t pack_mlen;
224
225 /* Table of objects we've written. */
226 static unsigned int object_entry_alloc = 1000;
@@ -616,7 +619,7 @@ static int store_object(
619 return 1;
620 }
621 e->type = type;
619 - e->offset = pack_offset;
622 + e->offset = pack_size;
623 object_count++;
624 object_count_by_type[type]++;
625
@@ -637,7 +640,7 @@ static int store_object(
640 hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
641 ywrite(pack_fd, hdr, hdrlen);
642 ywrite(pack_fd, last->sha1, sizeof(sha1));
640 - pack_offset += hdrlen + sizeof(sha1);
643 + pack_size += hdrlen + sizeof(sha1);
644 } else {
645 if (last)
646 last->depth = 0;
@@ -645,7 +648,7 @@ static int store_object(
648 s.avail_in = datlen;
649 hdrlen = encode_header(type, datlen, hdr);
650 ywrite(pack_fd, hdr, hdrlen);
648 - pack_offset += hdrlen;
651 + pack_size += hdrlen;
652 }
653
654 s.avail_out = deflateBound(&s, s.avail_in);
@@ -655,7 +658,7 @@ static int store_object(
658 deflateEnd(&s);
659
660 ywrite(pack_fd, out, s.total_out);
658 - pack_offset += s.total_out;
661 + pack_size += s.total_out;
662
663 free(out);
664 if (delta)
@@ -670,6 +673,127 @@ static int store_object(
673 return 0;
674 }
675
676 +static void* map_pack(unsigned long offset)
677 +{
678 + if (offset >= pack_size)
679 + die("object offset outside of pack file");
680 + if (offset >= pack_mlen) {
681 + if (pack_base)
682 + munmap(pack_base, pack_mlen);
683 + /* round out how much we map to 16 MB units */
684 + pack_mlen = pack_size;
685 + if (pack_mlen & ((1 << 24) - 1))
686 + pack_mlen = ((pack_mlen >> 24) + 1) << 24;
687 + pack_base = mmap(NULL,pack_mlen,PROT_READ,MAP_SHARED,pack_fd,0);
688 + if (pack_base == MAP_FAILED)
689 + die("Failed to map generated pack: %s", strerror(errno));
690 + remap_count++;
691 + }
692 + return (char*)pack_base + offset;
693 +}
694 +
695 +static unsigned long unpack_object_header(unsigned long offset,
696 + enum object_type *type,
697 + unsigned long *sizep)
698 +{
699 + unsigned shift;
700 + unsigned char c;
701 + unsigned long size;
702 +
703 + c = *(unsigned char*)map_pack(offset++);
704 + *type = (c >> 4) & 7;
705 + size = c & 15;
706 + shift = 4;
707 + while (c & 0x80) {
708 + c = *(unsigned char*)map_pack(offset++);
709 + size += (c & 0x7f) << shift;
710 + shift += 7;
711 + }
712 + *sizep = size;
713 + return offset;
714 +}
715 +
716 +static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
717 +{
718 + z_stream stream;
719 + unsigned char *result;
720 +
721 + result = xmalloc(sz + 1);
722 + result[sz] = 0;
723 +
724 + memset(&stream, 0, sizeof(stream));
725 + stream.next_in = map_pack(o);
726 + stream.avail_in = pack_mlen - o;
727 + stream.next_out = result;
728 + stream.avail_out = sz;
729 +
730 + inflateInit(&stream);
731 + for (;;) {
732 + int st = inflate(&stream, Z_FINISH);
733 + if (st == Z_STREAM_END)
734 + break;
735 + if (st == Z_OK) {
736 + o = stream.next_in - (unsigned char*)pack_base;
737 + stream.next_in = map_pack(o);
738 + stream.avail_in = pack_mlen - o;
739 + continue;
740 + }
741 + die("Error from zlib during inflate.");
742 + }
743 + inflateEnd(&stream);
744 + if (stream.total_out != sz)
745 + die("Error after inflate: sizes mismatch");
746 + return result;
747 +}
748 +
749 +static void *unpack_entry(unsigned long offset, unsigned long *sizep);
750 +
751 +static void *unpack_delta_entry(unsigned long offset,
752 + unsigned long delta_size,
753 + unsigned long *sizep)
754 +{
755 + struct object_entry *base_oe;
756 + unsigned char *base_sha1;
757 + void *delta_data, *base, *result;
758 + unsigned long base_size, result_size;
759 +
760 + base_sha1 = (unsigned char*)map_pack(offset + 20) - 20;
761 + base_oe = find_object(base_sha1);
762 + if (!base_oe)
763 + die("I'm broken; I can't find a base I know must be here.");
764 + base = unpack_entry(base_oe->offset, &base_size);
765 + delta_data = unpack_non_delta_entry(offset + 20, delta_size);
766 + result = patch_delta(base, base_size,
767 + delta_data, delta_size,
768 + &result_size);
769 + if (!result)
770 + die("failed to apply delta");
771 + free(delta_data);
772 + free(base);
773 + *sizep = result_size;
774 + return result;
775 +}
776 +
777 +static void *unpack_entry(unsigned long offset, unsigned long *sizep)
778 +{
779 + unsigned long size;
780 + enum object_type kind;
781 +
782 + offset = unpack_object_header(offset, &kind, &size);
783 + switch (kind) {
784 + case OBJ_DELTA:
785 + return unpack_delta_entry(offset, size, sizep);
786 + case OBJ_COMMIT:
787 + case OBJ_TREE:
788 + case OBJ_BLOB:
789 + case OBJ_TAG:
790 + *sizep = size;
791 + return unpack_non_delta_entry(offset, size);
792 + default:
793 + die("I created an object I can't read!");
794 + }
795 +}
796 +
797 static const char *get_mode(const char *str, unsigned int *modep)
798 {
799 unsigned char c;
@@ -691,7 +815,6 @@ static void load_tree(struct tree_entry *root)
815 unsigned long size;
816 char *buf;
817 const char *c;
694 - char type[20];
818
819 root->tree = t = new_tree_content(8);
820 if (!memcmp(root->sha1, null_sha1, 20))
@@ -699,11 +822,14 @@ static void load_tree(struct tree_entry *root)
822
823 myoe = find_object(root->sha1);
824 if (myoe) {
702 - die("FIXME");
825 + if (myoe->type != OBJ_TREE)
826 + die("Not a tree: %s", sha1_to_hex(root->sha1));
827 + buf = unpack_entry(myoe->offset, &size);
828 } else {
829 + char type[20];
830 buf = read_sha1_file(root->sha1, type, &size);
705 - if (!buf || strcmp(type, tree_type))
706 - die("Can't load existing tree %s", sha1_to_hex(root->sha1));
831 + if (!buf || !strcmp(type, tree_type))
832 + die("Can't load tree %s", sha1_to_hex(root->sha1));
833 }
834
835 c = buf;
@@ -880,7 +1006,7 @@ static void init_pack_header()
1006 hdr.hdr_entries = 0;
1007
1008 ywrite(pack_fd, &hdr, sizeof(hdr));
883 - pack_offset = sizeof(hdr);
1009 + pack_size = sizeof(hdr);
1010 }
1011
1012 static void fixup_header_footer()
@@ -1052,7 +1178,8 @@ static void cmd_new_blob()
1178
1179 static void unload_one_branch()
1180 {
1055 - while (cur_active_branches >= max_active_branches) {
1181 + while (cur_active_branches
1182 + && cur_active_branches >= max_active_branches) {
1183 unsigned long min_commit = ULONG_MAX;
1184 struct branch *e, *l = NULL, *p = NULL;
1185
@@ -1210,7 +1337,7 @@ static void cmd_new_commit()
1337 msg = cmd_data(&msglen);
1338
1339 /* ensure the branch is active/loaded */
1213 - if (!b->branch_tree.tree) {
1340 + if (!b->branch_tree.tree || !max_active_branches) {
1341 unload_one_branch();
1342 load_branch(b);
1343 }
@@ -1297,10 +1424,18 @@ static void cmd_new_branch()
1424 } else if (*from == ':') {
1425 unsigned long idnum = strtoul(from + 1, NULL, 10);
1426 struct object_entry *oe = find_mark(idnum);
1427 + unsigned long size;
1428 + char *buf;
1429 if (oe->type != OBJ_COMMIT)
1430 die("Mark :%lu not a commit", idnum);
1431 memcpy(b->sha1, oe->sha1, 20);
1303 - memcpy(b->branch_tree.sha1, null_sha1, 20);
1432 + buf = unpack_entry(oe->offset, &size);
1433 + if (!buf || size < 46)
1434 + die("Not a valid commit: %s", from);
1435 + if (memcmp("tree ", buf, 5)
1436 + || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1437 + die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1438 + free(buf);
1439 } else if (!get_sha1(from, b->sha1)) {
1440 if (!memcmp(b->sha1, null_sha1, 20))
1441 memcpy(b->branch_tree.sha1, null_sha1, 20);
@@ -1515,6 +1650,7 @@ int main(int argc, const char **argv)
1650 fprintf(stderr, "Memory total: %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1651 fprintf(stderr, " pools: %10lu KiB\n", total_allocd/1024);
1652 fprintf(stderr, " objects: %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1653 + fprintf(stderr, "Pack remaps: %10lu\n", remap_count);
1654 fprintf(stderr, "---------------------------------------------------\n");
1655
1656 stat(pack_name, &sb);