Implemented tree delta compression in fast-import.
We now store for every tree entry two modes and two sha1 values; the base (aka "version 0") and the current/new (aka "version 1"). When we generate a tree object we also regenerate the prior version object and use that as our base object for a delta. This strategy saves a significant amount of memory as we can continue to use the atom pool for file/directory names and only increases each tree entry by an additional 24 bytes of memory. Branches should automatically delta against their ancestor tree, unless the ancestor tree is already at the delta chain limit. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Aug 28, 2006 at 12:22 UTC
4cabf8583f934260697a065186f3dce135834ede
1 file changed
+155
-65
fast-import.c
+155
-65
@@ -132,7 +132,7 @@ struct mark_set
132
struct last_object
133
{
134
void *data;
135
- unsigned int len;
135
+ unsigned long len;
136
unsigned int depth;
137
unsigned char sha1[20];
138
};
@@ -157,14 +157,18 @@ struct tree_entry
157
{
158
struct tree_content *tree;
159
struct atom_str* name;
160
- unsigned int mode;
161
- unsigned char sha1[20];
160
+ struct tree_entry_ms
161
+ {
162
+ unsigned int mode;
163
+ unsigned char sha1[20];
164
+ } versions[2];
165
};
166
167
struct tree_content
168
{
169
unsigned int entry_capacity; /* must match avail_tree_content */
170
unsigned int entry_count;
171
+ unsigned int delta_depth;
172
struct tree_entry *entries[FLEX_ARRAY]; /* more */
173
};
174
@@ -203,6 +207,7 @@ static unsigned long duplicate_count;
207
static unsigned long marks_set_count;
208
static unsigned long object_count_by_type[9];
209
static unsigned long duplicate_count_by_type[9];
210
+static unsigned long delta_count_by_type[9];
211
212
/* Memory pools */
213
static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
@@ -224,7 +229,7 @@ static unsigned long pack_mlen = 128*1024*1024;
229
static unsigned long page_size;
230
231
/* Table of objects we've written. */
227
-static unsigned int object_entry_alloc = 1000;
232
+static unsigned int object_entry_alloc = 5000;
233
static struct object_entry_pool *blocks;
234
static struct object_entry *object_table[1 << 16];
235
static struct mark_set *marks;
@@ -486,6 +491,7 @@ static struct tree_content* new_tree_content(unsigned int cnt)
491
492
t = (struct tree_content*)f;
493
t->entry_count = 0;
494
+ t->delta_depth = 0;
495
return t;
496
}
497
@@ -512,6 +518,7 @@ static struct tree_content* grow_tree_content(
518
{
519
struct tree_content *r = new_tree_content(t->entry_count + amt);
520
r->entry_count = t->entry_count;
521
+ r->delta_depth = t->delta_depth;
522
memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
523
release_tree_content(t);
524
return r;
@@ -642,6 +649,7 @@ static int store_object(
649
deflateInit(&s, zlib_compression_level);
650
651
if (delta) {
652
+ delta_count_by_type[type]++;
653
last->depth++;
654
s.next_in = delta;
655
s.avail_in = deltalen;
@@ -755,11 +763,14 @@ static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
763
return result;
764
}
765
758
-static void *unpack_entry(unsigned long offset, unsigned long *sizep);
766
+static void *unpack_entry(unsigned long offset,
767
+ unsigned long *sizep,
768
+ unsigned int *delta_depth);
769
770
static void *unpack_delta_entry(unsigned long offset,
771
unsigned long delta_size,
762
- unsigned long *sizep)
772
+ unsigned long *sizep,
773
+ unsigned int *delta_depth)
774
{
775
struct object_entry *base_oe;
776
unsigned char *base_sha1;
@@ -770,7 +781,7 @@ static void *unpack_delta_entry(unsigned long offset,
781
base_oe = find_object(base_sha1);
782
if (!base_oe)
783
die("I'm broken; I can't find a base I know must be here.");
773
- base = unpack_entry(base_oe->offset, &base_size);
784
+ base = unpack_entry(base_oe->offset, &base_size, delta_depth);
785
delta_data = unpack_non_delta_entry(offset + 20, delta_size);
786
result = patch_delta(base, base_size,
787
delta_data, delta_size,
@@ -780,10 +791,13 @@ static void *unpack_delta_entry(unsigned long offset,
791
free(delta_data);
792
free(base);
793
*sizep = result_size;
794
+ (*delta_depth)++;
795
return result;
796
}
797
786
-static void *unpack_entry(unsigned long offset, unsigned long *sizep)
798
+static void *unpack_entry(unsigned long offset,
799
+ unsigned long *sizep,
800
+ unsigned int *delta_depth)
801
{
802
unsigned long size;
803
enum object_type kind;
@@ -791,12 +805,13 @@ static void *unpack_entry(unsigned long offset, unsigned long *sizep)
805
offset = unpack_object_header(offset, &kind, &size);
806
switch (kind) {
807
case OBJ_DELTA:
794
- return unpack_delta_entry(offset, size, sizep);
808
+ return unpack_delta_entry(offset, size, sizep, delta_depth);
809
case OBJ_COMMIT:
810
case OBJ_TREE:
811
case OBJ_BLOB:
812
case OBJ_TAG:
813
*sizep = size;
814
+ *delta_depth = 0;
815
return unpack_non_delta_entry(offset, size);
816
default:
817
die("I created an object I can't read!");
@@ -819,6 +834,7 @@ static const char *get_mode(const char *str, unsigned int *modep)
834
835
static void load_tree(struct tree_entry *root)
836
{
837
+ unsigned char* sha1 = root->versions[1].sha1;
838
struct object_entry *myoe;
839
struct tree_content *t;
840
unsigned long size;
@@ -826,19 +842,19 @@ static void load_tree(struct tree_entry *root)
842
const char *c;
843
844
root->tree = t = new_tree_content(8);
829
- if (is_null_sha1(root->sha1))
845
+ if (is_null_sha1(sha1))
846
return;
847
832
- myoe = find_object(root->sha1);
848
+ myoe = find_object(sha1);
849
if (myoe) {
850
if (myoe->type != OBJ_TREE)
835
- die("Not a tree: %s", sha1_to_hex(root->sha1));
836
- buf = unpack_entry(myoe->offset, &size);
851
+ die("Not a tree: %s", sha1_to_hex(sha1));
852
+ buf = unpack_entry(myoe->offset, &size, &t->delta_depth);
853
} else {
854
char type[20];
839
- buf = read_sha1_file(root->sha1, type, &size);
855
+ buf = read_sha1_file(sha1, type, &size);
856
if (!buf || strcmp(type, tree_type))
841
- die("Can't load tree %s", sha1_to_hex(root->sha1));
857
+ die("Can't load tree %s", sha1_to_hex(sha1));
858
}
859
860
c = buf;
@@ -850,56 +866,116 @@ static void load_tree(struct tree_entry *root)
866
t->entries[t->entry_count++] = e;
867
868
e->tree = NULL;
853
- c = get_mode(c, &e->mode);
869
+ c = get_mode(c, &e->versions[1].mode);
870
if (!c)
855
- die("Corrupt mode in %s", sha1_to_hex(root->sha1));
871
+ die("Corrupt mode in %s", sha1_to_hex(sha1));
872
+ e->versions[0].mode = e->versions[1].mode;
873
e->name = to_atom(c, strlen(c));
874
c += e->name->str_len + 1;
858
- hashcpy(e->sha1, c);
875
+ hashcpy(e->versions[0].sha1, (unsigned char*)c);
876
+ hashcpy(e->versions[1].sha1, (unsigned char*)c);
877
c += 20;
878
}
879
free(buf);
880
}
881
864
-static int tecmp (const void *_a, const void *_b)
882
+static int tecmp0 (const void *_a, const void *_b)
883
{
884
struct tree_entry *a = *((struct tree_entry**)_a);
885
struct tree_entry *b = *((struct tree_entry**)_b);
886
return base_name_compare(
869
- a->name->str_dat, a->name->str_len, a->mode,
870
- b->name->str_dat, b->name->str_len, b->mode);
887
+ a->name->str_dat, a->name->str_len, a->versions[0].mode,
888
+ b->name->str_dat, b->name->str_len, b->versions[0].mode);
889
}
890
873
-static void store_tree(struct tree_entry *root)
891
+static int tecmp1 (const void *_a, const void *_b)
892
{
875
- struct tree_content *t = root->tree;
893
+ struct tree_entry *a = *((struct tree_entry**)_a);
894
+ struct tree_entry *b = *((struct tree_entry**)_b);
895
+ return base_name_compare(
896
+ a->name->str_dat, a->name->str_len, a->versions[1].mode,
897
+ b->name->str_dat, b->name->str_len, b->versions[1].mode);
898
+}
899
+
900
+static void* mktree(struct tree_content *t, int v, unsigned long *szp)
901
+{
902
+ size_t maxlen = 0;
903
unsigned int i;
877
- size_t maxlen;
904
char *buf, *c;
905
880
- if (!is_null_sha1(root->sha1))
881
- return;
906
+ if (!v)
907
+ qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp0);
908
+ else
909
+ qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp1);
910
883
- maxlen = 0;
911
for (i = 0; i < t->entry_count; i++) {
885
- maxlen += t->entries[i]->name->str_len + 34;
886
- if (t->entries[i]->tree)
887
- store_tree(t->entries[i]);
912
+ if (t->entries[i]->versions[v].mode)
913
+ maxlen += t->entries[i]->name->str_len + 34;
914
}
915
890
- qsort(t->entries, t->entry_count, sizeof(t->entries[0]), tecmp);
916
buf = c = xmalloc(maxlen);
917
for (i = 0; i < t->entry_count; i++) {
918
struct tree_entry *e = t->entries[i];
894
- c += sprintf(c, "%o", e->mode);
919
+ if (!e->versions[v].mode)
920
+ continue;
921
+ c += sprintf(c, "%o", e->versions[v].mode);
922
*c++ = ' ';
923
strcpy(c, e->name->str_dat);
924
c += e->name->str_len + 1;
898
- hashcpy(c, e->sha1);
925
+ hashcpy((unsigned char*)c, e->versions[v].sha1);
926
c += 20;
927
}
901
- store_object(OBJ_TREE, buf, c - buf, NULL, root->sha1, 0);
902
- free(buf);
928
+
929
+ *szp = c - buf;
930
+ return buf;
931
+}
932
+
933
+static void store_tree(struct tree_entry *root)
934
+{
935
+ struct tree_content *t = root->tree;
936
+ unsigned int i, j, del;
937
+ unsigned long vers1len;
938
+ void **vers1dat;
939
+ struct last_object lo;
940
+
941
+ if (!is_null_sha1(root->versions[1].sha1))
942
+ return;
943
+
944
+ for (i = 0; i < t->entry_count; i++) {
945
+ if (t->entries[i]->tree)
946
+ store_tree(t->entries[i]);
947
+ }
948
+
949
+ if (is_null_sha1(root->versions[0].sha1)
950
+ || !find_object(root->versions[0].sha1)) {
951
+ lo.data = NULL;
952
+ lo.depth = 0;
953
+ } else {
954
+ lo.data = mktree(t, 0, &lo.len);
955
+ lo.depth = t->delta_depth;
956
+ hashcpy(lo.sha1, root->versions[0].sha1);
957
+ }
958
+ vers1dat = mktree(t, 1, &vers1len);
959
+
960
+ store_object(OBJ_TREE, vers1dat, vers1len,
961
+ &lo, root->versions[1].sha1, 0);
962
+ /* note: lo.dat (if created) was freed by store_object */
963
+ free(vers1dat);
964
+
965
+ t->delta_depth = lo.depth;
966
+ hashcpy(root->versions[0].sha1, root->versions[1].sha1);
967
+ for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
968
+ struct tree_entry *e = t->entries[i];
969
+ if (e->versions[1].mode) {
970
+ e->versions[0].mode = e->versions[1].mode;
971
+ hashcpy(e->versions[0].sha1, e->versions[1].sha1);
972
+ t->entries[j++] = e;
973
+ } else {
974
+ release_tree_entry(e);
975
+ del++;
976
+ }
977
+ }
978
+ t->entry_count -= del;
979
}
980
981
static int tree_content_set(
@@ -923,25 +999,26 @@ static int tree_content_set(
999
e = t->entries[i];
1000
if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
1001
if (!slash1) {
926
- if (e->mode == mode && !hashcmp(e->sha1, sha1))
1002
+ if (e->versions[1].mode == mode
1003
+ && !hashcmp(e->versions[1].sha1, sha1))
1004
return 0;
928
- e->mode = mode;
929
- hashcpy(e->sha1, sha1);
1005
+ e->versions[1].mode = mode;
1006
+ hashcpy(e->versions[1].sha1, sha1);
1007
if (e->tree) {
1008
release_tree_content_recursive(e->tree);
1009
e->tree = NULL;
1010
}
934
- hashclr(root->sha1);
1011
+ hashclr(root->versions[1].sha1);
1012
return 1;
1013
}
937
- if (!S_ISDIR(e->mode)) {
1014
+ if (!S_ISDIR(e->versions[1].mode)) {
1015
e->tree = new_tree_content(8);
939
- e->mode = S_IFDIR;
1016
+ e->versions[1].mode = S_IFDIR;
1017
}
1018
if (!e->tree)
1019
load_tree(e);
1020
if (tree_content_set(e, slash1 + 1, sha1, mode)) {
944
- hashclr(root->sha1);
1021
+ hashclr(root->versions[1].sha1);
1022
return 1;
1023
}
1024
return 0;
@@ -952,17 +1029,19 @@ static int tree_content_set(
1029
root->tree = t = grow_tree_content(t, 8);
1030
e = new_tree_entry();
1031
e->name = to_atom(p, n);
1032
+ e->versions[0].mode = 0;
1033
+ hashclr(e->versions[0].sha1);
1034
t->entries[t->entry_count++] = e;
1035
if (slash1) {
1036
e->tree = new_tree_content(8);
958
- e->mode = S_IFDIR;
1037
+ e->versions[1].mode = S_IFDIR;
1038
tree_content_set(e, slash1 + 1, sha1, mode);
1039
} else {
1040
e->tree = NULL;
962
- e->mode = mode;
963
- hashcpy(e->sha1, sha1);
1041
+ e->versions[1].mode = mode;
1042
+ hashcpy(e->versions[1].sha1, sha1);
1043
}
965
- hashclr(root->sha1);
1044
+ hashclr(root->versions[1].sha1);
1045
return 1;
1046
}
1047
@@ -982,14 +1061,14 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
1061
for (i = 0; i < t->entry_count; i++) {
1062
e = t->entries[i];
1063
if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
985
- if (!slash1 || !S_ISDIR(e->mode))
1064
+ if (!slash1 || !S_ISDIR(e->versions[1].mode))
1065
goto del_entry;
1066
if (!e->tree)
1067
load_tree(e);
1068
if (tree_content_remove(e, slash1 + 1)) {
1069
if (!e->tree->entry_count)
1070
goto del_entry;
992
- hashclr(root->sha1);
1071
+ hashclr(root->versions[1].sha1);
1072
return 1;
1073
}
1074
return 0;
@@ -998,11 +1077,13 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
1077
return 0;
1078
1079
del_entry:
1001
- for (i++; i < t->entry_count; i++)
1002
- t->entries[i-1] = t->entries[i];
1003
- t->entry_count--;
1004
- release_tree_entry(e);
1005
- hashclr(root->sha1);
1080
+ if (e->tree) {
1081
+ release_tree_content_recursive(e->tree);
1082
+ e->tree = NULL;
1083
+ }
1084
+ e->versions[1].mode = 0;
1085
+ hashclr(e->versions[1].sha1);
1086
+ hashclr(root->versions[1].sha1);
1087
return 1;
1088
}
1089
@@ -1359,27 +1440,33 @@ static void cmd_from(struct branch *b)
1440
if (b == s)
1441
die("Can't create a branch from itself: %s", b->name);
1442
else if (s) {
1443
+ unsigned char *t = s->branch_tree.versions[1].sha1;
1444
hashcpy(b->sha1, s->sha1);
1363
- hashcpy(b->branch_tree.sha1, s->branch_tree.sha1);
1445
+ hashcpy(b->branch_tree.versions[0].sha1, t);
1446
+ hashcpy(b->branch_tree.versions[1].sha1, t);
1447
} else if (*from == ':') {
1448
unsigned long idnum = strtoul(from + 1, NULL, 10);
1449
struct object_entry *oe = find_mark(idnum);
1450
unsigned long size;
1451
+ unsigned int depth;
1452
char *buf;
1453
if (oe->type != OBJ_COMMIT)
1454
die("Mark :%lu not a commit", idnum);
1455
hashcpy(b->sha1, oe->sha1);
1372
- buf = unpack_entry(oe->offset, &size);
1456
+ buf = unpack_entry(oe->offset, &size, &depth);
1457
if (!buf || size < 46)
1458
die("Not a valid commit: %s", from);
1459
if (memcmp("tree ", buf, 5)
1376
- || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1460
+ || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
1461
die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1462
free(buf);
1463
+ hashcpy(b->branch_tree.versions[0].sha1,
1464
+ b->branch_tree.versions[1].sha1);
1465
} else if (!get_sha1(from, b->sha1)) {
1380
- if (is_null_sha1(b->sha1))
1381
- hashclr(b->branch_tree.sha1);
1382
- else {
1466
+ if (is_null_sha1(b->sha1)) {
1467
+ hashclr(b->branch_tree.versions[0].sha1);
1468
+ hashclr(b->branch_tree.versions[1].sha1);
1469
+ } else {
1470
unsigned long size;
1471
char *buf;
1472
@@ -1388,9 +1475,11 @@ static void cmd_from(struct branch *b)
1475
if (!buf || size < 46)
1476
die("Not a valid commit: %s", from);
1477
if (memcmp("tree ", buf, 5)
1391
- || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1478
+ || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
1479
die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1480
free(buf);
1481
+ hashcpy(b->branch_tree.versions[0].sha1,
1482
+ b->branch_tree.versions[1].sha1);
1483
}
1484
} else
1485
die("Invalid ref name or SHA1 expression: %s", from);
@@ -1466,7 +1555,8 @@ static void cmd_new_commit()
1555
? strlen(author) + strlen(committer)
1556
: 2 * strlen(committer)));
1557
sp = body;
1469
- sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
1558
+ sp += sprintf(sp, "tree %s\n",
1559
+ sha1_to_hex(b->branch_tree.versions[1].sha1));
1560
if (!is_null_sha1(b->sha1))
1561
sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
1562
if (author)
@@ -1722,10 +1812,10 @@ int main(int argc, const char **argv)
1812
fprintf(stderr, "---------------------------------------------------\n");
1813
fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1814
fprintf(stderr, "Total objects: %10lu (%10lu duplicates)\n", object_count, duplicate_count);
1725
- fprintf(stderr, " blobs : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB]);
1726
- fprintf(stderr, " trees : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE]);
1727
- fprintf(stderr, " commits: %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT]);
1728
- fprintf(stderr, " tags : %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG]);
1815
+ fprintf(stderr, " blobs : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1816
+ fprintf(stderr, " trees : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1817
+ fprintf(stderr, " commits: %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1818
+ fprintf(stderr, " tags : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1819
fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
1820
fprintf(stderr, " marks: %10u (%10lu unique )\n", (1 << marks->shift) * 1024, marks_set_count);
1821
fprintf(stderr, " atoms: %10u\n", atom_cnt);