@matheusbsbs / matheus-bagliano / commits / 4cabf8583f
Fork 0

Implemented tree delta compression in fast-import.

We now store for every tree entry two modes and two sha1 values; the base (aka "version 0") and the current/new (aka "version 1"). When we generate a tree object we also regenerate the prior version object and use that as our base object for a delta. This strategy saves a significant amount of memory as we can continue to use the atom pool for file/directory names and only increases each tree entry by an additional 24 bytes of memory. Branches should automatically delta against their ancestor tree, unless the ancestor tree is already at the delta chain limit. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Aug 28, 2006 at 12:22 UTC 4cabf8583f934260697a065186f3dce135834ede
1 file changed +155 -65
fast-import.c
+155 -65
@@ -132,7 +132,7 @@ struct mark_set
132 struct last_object
133 {
134 void *data;
135 - unsigned int len;
135 + unsigned long len;
136 unsigned int depth;
137 unsigned char sha1[20];
138 };
@@ -157,14 +157,18 @@ struct tree_entry
157 {
158 struct tree_content *tree;
159 struct atom_str* name;
160 - unsigned int mode;
161 - unsigned char sha1[20];
160 + struct tree_entry_ms
161 + {
162 + unsigned int mode;
163 + unsigned char sha1[20];
164 + } versions[2];
165 };
166
167 struct tree_content
168 {
169 unsigned int entry_capacity; /* must match avail_tree_content */
170 unsigned int entry_count;
171 + unsigned int delta_depth;
172 struct tree_entry *entries[FLEX_ARRAY]; /* more */
173 };
174
@@ -203,6 +207,7 @@ static unsigned long duplicate_count;
207 static unsigned long marks_set_count;
208 static unsigned long object_count_by_type[9];
209 static unsigned long duplicate_count_by_type[9];
210 +static unsigned long delta_count_by_type[9];
211
212 /* Memory pools */
213 static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
@@ -224,7 +229,7 @@ static unsigned long pack_mlen = 128*1024*1024;
229 static unsigned long page_size;
230
231 /* Table of objects we've written. */
227 -static unsigned int object_entry_alloc = 1000;
232 +static unsigned int object_entry_alloc = 5000;
233 static struct object_entry_pool *blocks;
234 static struct object_entry *object_table[1 << 16];
235 static struct mark_set *marks;
@@ -486,6 +491,7 @@ static struct tree_content* new_tree_content(unsigned int cnt)
491
492 t = (struct tree_content*)f;
493 t->entry_count = 0;
494 + t->delta_depth = 0;
495 return t;
496 }
497
@@ -512,6 +518,7 @@ static struct tree_content* grow_tree_content(
518 {
519 struct tree_content *r = new_tree_content(t->entry_count + amt);
520 r->entry_count = t->entry_count;
521 + r->delta_depth = t->delta_depth;
522 memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
523 release_tree_content(t);
524 return r;
@@ -642,6 +649,7 @@ static int store_object(
649 deflateInit(&s, zlib_compression_level);
650
651 if (delta) {
652 + delta_count_by_type[type]++;
653 last->depth++;
654 s.next_in = delta;
655 s.avail_in = deltalen;
@@ -755,11 +763,14 @@ static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
763 return result;
764 }
765
758 -static void *unpack_entry(unsigned long offset, unsigned long *sizep);
766 +static void *unpack_entry(unsigned long offset,
767 + unsigned long *sizep,
768 + unsigned int *delta_depth);
769
770 static void *unpack_delta_entry(unsigned long offset,
771 unsigned long delta_size,
762 - unsigned long *sizep)
772 + unsigned long *sizep,
773 + unsigned int *delta_depth)
774 {
775 struct object_entry *base_oe;
776 unsigned char *base_sha1;
@@ -770,7 +781,7 @@ static void *unpack_delta_entry(unsigned long offset,
781 base_oe = find_object(base_sha1);
782 if (!base_oe)
783 die("I'm broken; I can't find a base I know must be here.");
773 - base = unpack_entry(base_oe->offset, &base_size);
784 + base = unpack_entry(base_oe->offset, &base_size, delta_depth);
785 delta_data = unpack_non_delta_entry(offset + 20, delta_size);
786 result = patch_delta(base, base_size,
787 delta_data, delta_size,
@@ -780,10 +791,13 @@ static void *unpack_delta_entry(unsigned long offset,
791 free(delta_data);
792 free(base);
793 *sizep = result_size;
794 + (*delta_depth)++;
795 return result;
796 }
797
786 -static void *unpack_entry(unsigned long offset, unsigned long *sizep)
798 +static void *unpack_entry(unsigned long offset,
799 + unsigned long *sizep,
800 + unsigned int *delta_depth)
801 {
802 unsigned long size;
803 enum object_type kind;
@@ -791,12 +805,13 @@ static void *unpack_entry(unsigned long offset, unsigned long *sizep)
805 offset = unpack_object_header(offset, &kind, &size);
806 switch (kind) {
807 case OBJ_DELTA:
794 - return unpack_delta_entry(offset, size, sizep);
808 + return unpack_delta_entry(offset, size, sizep, delta_depth);
809 case OBJ_COMMIT:
810 case OBJ_TREE:
811 case OBJ_BLOB:
812 case OBJ_TAG:
813 *sizep = size;
814 + *delta_depth = 0;
815 return unpack_non_delta_entry(offset, size);
816 default:
817 die("I created an object I can't read!");
@@ -819,6 +834,7 @@ static const char *get_mode(const char *str, unsigned int *modep)
834
835 static void load_tree(struct tree_entry *root)
836 {
837 + unsigned char* sha1 = root->versions[1].sha1;
838 struct object_entry *myoe;
839 struct tree_content *t;
840 unsigned long size;
@@ -826,19 +842,19 @@ static void load_tree(struct tree_entry *root)
842 const char *c;
843
844 root->tree = t = new_tree_content(8);
829 - if (is_null_sha1(root->sha1))
845 + if (is_null_sha1(sha1))
846 return;
847
832 - myoe = find_object(root->sha1);
848 + myoe = find_object(sha1);
849 if (myoe) {
850 if (myoe->type != OBJ_TREE)
835 - die("Not a tree: %s", sha1_to_hex(root->sha1));
836 - buf = unpack_entry(myoe->offset, &size);
851 + die("Not a tree: %s", sha1_to_hex(sha1));
852 + buf = unpack_entry(myoe->offset, &size, &t->delta_depth);
853 } else {
854 char type[20];
839 - buf = read_sha1_file(root->sha1, type, &size);
855 + buf = read_sha1_file(sha1, type, &size);
856 if (!buf || strcmp(type, tree_type))
841 - die("Can't load tree %s", sha1_to_hex(root->sha1));
857 + die("Can't load tree %s", sha1_to_hex(sha1));
858 }
859
860 c = buf;
@@ -850,56 +866,116 @@ static void load_tree(struct tree_entry *root)
866 t->entries[t->entry_count++] = e;
867
868 e->tree = NULL;
853 - c = get_mode(c, &e->mode);
869 + c = get_mode(c, &e->versions[1].mode);
870 if (!c)
855 - die("Corrupt mode in %s", sha1_to_hex(root->sha1));
871 + die("Corrupt mode in %s", sha1_to_hex(sha1));
872 + e->versions[0].mode = e->versions[1].mode;
873 e->name = to_atom(c, strlen(c));
874 c += e->name->str_len + 1;
858 - hashcpy(e->sha1, c);
875 + hashcpy(e->versions[0].sha1, (unsigned char*)c);
876 + hashcpy(e->versions[1].sha1, (unsigned char*)c);
877 c += 20;
878 }
879 free(buf);
880 }
881
864 -static int tecmp (const void *_a, const void *_b)
882 +static int tecmp0 (const void *_a, const void *_b)
883 {
884 struct tree_entry *a = *((struct tree_entry**)_a);
885 struct tree_entry *b = *((struct tree_entry**)_b);
886 return base_name_compare(
869 - a->name->str_dat, a->name->str_len, a->mode,
870 - b->name->str_dat, b->name->str_len, b->mode);
887 + a->name->str_dat, a->name->str_len, a->versions[0].mode,
888 + b->name->str_dat, b->name->str_len, b->versions[0].mode);
889 }
890
873 -static void store_tree(struct tree_entry *root)
891 +static int tecmp1 (const void *_a, const void *_b)
892 {
875 - struct tree_content *t = root->tree;
893 + struct tree_entry *a = *((struct tree_entry**)_a);
894 + struct tree_entry *b = *((struct tree_entry**)_b);
895 + return base_name_compare(
896 + a->name->str_dat, a->name->str_len, a->versions[1].mode,
897 + b->name->str_dat, b->name->str_len, b->versions[1].mode);
898 +}
899 +
900 +static void* mktree(struct tree_content *t, int v, unsigned long *szp)
901 +{
902 + size_t maxlen = 0;
903 unsigned int i;
877 - size_t maxlen;
904 char *buf, *c;
905
880 - if (!is_null_sha1(root->sha1))
881 - return;
906 + if (!v)
907 + qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp0);
908 + else
909 + qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp1);
910
883 - maxlen = 0;
911 for (i = 0; i < t->entry_count; i++) {
885 - maxlen += t->entries[i]->name->str_len + 34;
886 - if (t->entries[i]->tree)
887 - store_tree(t->entries[i]);
912 + if (t->entries[i]->versions[v].mode)
913 + maxlen += t->entries[i]->name->str_len + 34;
914 }
915
890 - qsort(t->entries, t->entry_count, sizeof(t->entries[0]), tecmp);
916 buf = c = xmalloc(maxlen);
917 for (i = 0; i < t->entry_count; i++) {
918 struct tree_entry *e = t->entries[i];
894 - c += sprintf(c, "%o", e->mode);
919 + if (!e->versions[v].mode)
920 + continue;
921 + c += sprintf(c, "%o", e->versions[v].mode);
922 *c++ = ' ';
923 strcpy(c, e->name->str_dat);
924 c += e->name->str_len + 1;
898 - hashcpy(c, e->sha1);
925 + hashcpy((unsigned char*)c, e->versions[v].sha1);
926 c += 20;
927 }
901 - store_object(OBJ_TREE, buf, c - buf, NULL, root->sha1, 0);
902 - free(buf);
928 +
929 + *szp = c - buf;
930 + return buf;
931 +}
932 +
933 +static void store_tree(struct tree_entry *root)
934 +{
935 + struct tree_content *t = root->tree;
936 + unsigned int i, j, del;
937 + unsigned long vers1len;
938 + void **vers1dat;
939 + struct last_object lo;
940 +
941 + if (!is_null_sha1(root->versions[1].sha1))
942 + return;
943 +
944 + for (i = 0; i < t->entry_count; i++) {
945 + if (t->entries[i]->tree)
946 + store_tree(t->entries[i]);
947 + }
948 +
949 + if (is_null_sha1(root->versions[0].sha1)
950 + || !find_object(root->versions[0].sha1)) {
951 + lo.data = NULL;
952 + lo.depth = 0;
953 + } else {
954 + lo.data = mktree(t, 0, &lo.len);
955 + lo.depth = t->delta_depth;
956 + hashcpy(lo.sha1, root->versions[0].sha1);
957 + }
958 + vers1dat = mktree(t, 1, &vers1len);
959 +
960 + store_object(OBJ_TREE, vers1dat, vers1len,
961 + &lo, root->versions[1].sha1, 0);
962 + /* note: lo.dat (if created) was freed by store_object */
963 + free(vers1dat);
964 +
965 + t->delta_depth = lo.depth;
966 + hashcpy(root->versions[0].sha1, root->versions[1].sha1);
967 + for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
968 + struct tree_entry *e = t->entries[i];
969 + if (e->versions[1].mode) {
970 + e->versions[0].mode = e->versions[1].mode;
971 + hashcpy(e->versions[0].sha1, e->versions[1].sha1);
972 + t->entries[j++] = e;
973 + } else {
974 + release_tree_entry(e);
975 + del++;
976 + }
977 + }
978 + t->entry_count -= del;
979 }
980
981 static int tree_content_set(
@@ -923,25 +999,26 @@ static int tree_content_set(
999 e = t->entries[i];
1000 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
1001 if (!slash1) {
926 - if (e->mode == mode && !hashcmp(e->sha1, sha1))
1002 + if (e->versions[1].mode == mode
1003 + && !hashcmp(e->versions[1].sha1, sha1))
1004 return 0;
928 - e->mode = mode;
929 - hashcpy(e->sha1, sha1);
1005 + e->versions[1].mode = mode;
1006 + hashcpy(e->versions[1].sha1, sha1);
1007 if (e->tree) {
1008 release_tree_content_recursive(e->tree);
1009 e->tree = NULL;
1010 }
934 - hashclr(root->sha1);
1011 + hashclr(root->versions[1].sha1);
1012 return 1;
1013 }
937 - if (!S_ISDIR(e->mode)) {
1014 + if (!S_ISDIR(e->versions[1].mode)) {
1015 e->tree = new_tree_content(8);
939 - e->mode = S_IFDIR;
1016 + e->versions[1].mode = S_IFDIR;
1017 }
1018 if (!e->tree)
1019 load_tree(e);
1020 if (tree_content_set(e, slash1 + 1, sha1, mode)) {
944 - hashclr(root->sha1);
1021 + hashclr(root->versions[1].sha1);
1022 return 1;
1023 }
1024 return 0;
@@ -952,17 +1029,19 @@ static int tree_content_set(
1029 root->tree = t = grow_tree_content(t, 8);
1030 e = new_tree_entry();
1031 e->name = to_atom(p, n);
1032 + e->versions[0].mode = 0;
1033 + hashclr(e->versions[0].sha1);
1034 t->entries[t->entry_count++] = e;
1035 if (slash1) {
1036 e->tree = new_tree_content(8);
958 - e->mode = S_IFDIR;
1037 + e->versions[1].mode = S_IFDIR;
1038 tree_content_set(e, slash1 + 1, sha1, mode);
1039 } else {
1040 e->tree = NULL;
962 - e->mode = mode;
963 - hashcpy(e->sha1, sha1);
1041 + e->versions[1].mode = mode;
1042 + hashcpy(e->versions[1].sha1, sha1);
1043 }
965 - hashclr(root->sha1);
1044 + hashclr(root->versions[1].sha1);
1045 return 1;
1046 }
1047
@@ -982,14 +1061,14 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
1061 for (i = 0; i < t->entry_count; i++) {
1062 e = t->entries[i];
1063 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
985 - if (!slash1 || !S_ISDIR(e->mode))
1064 + if (!slash1 || !S_ISDIR(e->versions[1].mode))
1065 goto del_entry;
1066 if (!e->tree)
1067 load_tree(e);
1068 if (tree_content_remove(e, slash1 + 1)) {
1069 if (!e->tree->entry_count)
1070 goto del_entry;
992 - hashclr(root->sha1);
1071 + hashclr(root->versions[1].sha1);
1072 return 1;
1073 }
1074 return 0;
@@ -998,11 +1077,13 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
1077 return 0;
1078
1079 del_entry:
1001 - for (i++; i < t->entry_count; i++)
1002 - t->entries[i-1] = t->entries[i];
1003 - t->entry_count--;
1004 - release_tree_entry(e);
1005 - hashclr(root->sha1);
1080 + if (e->tree) {
1081 + release_tree_content_recursive(e->tree);
1082 + e->tree = NULL;
1083 + }
1084 + e->versions[1].mode = 0;
1085 + hashclr(e->versions[1].sha1);
1086 + hashclr(root->versions[1].sha1);
1087 return 1;
1088 }
1089
@@ -1359,27 +1440,33 @@ static void cmd_from(struct branch *b)
1440 if (b == s)
1441 die("Can't create a branch from itself: %s", b->name);
1442 else if (s) {
1443 + unsigned char *t = s->branch_tree.versions[1].sha1;
1444 hashcpy(b->sha1, s->sha1);
1363 - hashcpy(b->branch_tree.sha1, s->branch_tree.sha1);
1445 + hashcpy(b->branch_tree.versions[0].sha1, t);
1446 + hashcpy(b->branch_tree.versions[1].sha1, t);
1447 } else if (*from == ':') {
1448 unsigned long idnum = strtoul(from + 1, NULL, 10);
1449 struct object_entry *oe = find_mark(idnum);
1450 unsigned long size;
1451 + unsigned int depth;
1452 char *buf;
1453 if (oe->type != OBJ_COMMIT)
1454 die("Mark :%lu not a commit", idnum);
1455 hashcpy(b->sha1, oe->sha1);
1372 - buf = unpack_entry(oe->offset, &size);
1456 + buf = unpack_entry(oe->offset, &size, &depth);
1457 if (!buf || size < 46)
1458 die("Not a valid commit: %s", from);
1459 if (memcmp("tree ", buf, 5)
1376 - || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1460 + || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
1461 die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1462 free(buf);
1463 + hashcpy(b->branch_tree.versions[0].sha1,
1464 + b->branch_tree.versions[1].sha1);
1465 } else if (!get_sha1(from, b->sha1)) {
1380 - if (is_null_sha1(b->sha1))
1381 - hashclr(b->branch_tree.sha1);
1382 - else {
1466 + if (is_null_sha1(b->sha1)) {
1467 + hashclr(b->branch_tree.versions[0].sha1);
1468 + hashclr(b->branch_tree.versions[1].sha1);
1469 + } else {
1470 unsigned long size;
1471 char *buf;
1472
@@ -1388,9 +1475,11 @@ static void cmd_from(struct branch *b)
1475 if (!buf || size < 46)
1476 die("Not a valid commit: %s", from);
1477 if (memcmp("tree ", buf, 5)
1391 - || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1478 + || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
1479 die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1480 free(buf);
1481 + hashcpy(b->branch_tree.versions[0].sha1,
1482 + b->branch_tree.versions[1].sha1);
1483 }
1484 } else
1485 die("Invalid ref name or SHA1 expression: %s", from);
@@ -1466,7 +1555,8 @@ static void cmd_new_commit()
1555 ? strlen(author) + strlen(committer)
1556 : 2 * strlen(committer)));
1557 sp = body;
1469 - sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
1558 + sp += sprintf(sp, "tree %s\n",
1559 + sha1_to_hex(b->branch_tree.versions[1].sha1));
1560 if (!is_null_sha1(b->sha1))
1561 sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
1562 if (author)
@@ -1722,10 +1812,10 @@ int main(int argc, const char **argv)
1812 fprintf(stderr, "---------------------------------------------------\n");
1813 fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1814 fprintf(stderr, "Total objects: %10lu (%10lu duplicates)\n", object_count, duplicate_count);
1725 - fprintf(stderr, " blobs : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB]);
1726 - fprintf(stderr, " trees : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE]);
1727 - fprintf(stderr, " commits: %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT]);
1728 - fprintf(stderr, " tags : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG]);
1815 + fprintf(stderr, " blobs : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1816 + fprintf(stderr, " trees : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1817 + fprintf(stderr, " commits: %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1818 + fprintf(stderr, " tags : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1819 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
1820 fprintf(stderr, " marks: %10u (%10lu unique )\n", (1 << marks->shift) * 1024, marks_set_count);
1821 fprintf(stderr, " atoms: %10u\n", atom_cnt);