Implemented automatic checkpoints within fast-import.
When the number of objects or number of bytes gets close to the limit allowed by the packfile format (or configured on the command line by our caller) we should automatically checkpoint the current packfile and start a new one before writing the object out. This does however require that we abandon the delta (if we had one) as its not valid in a new packfile. I also added the simple rule that if we got a delta back but the delta itself is the same size as or larger than the uncompressed object to ignore the delta and just store the object data. This should avoid some really bad behavior caused by our current delta strategy. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Jan 15, 2007 at 08:00 UTC
d9ee53ce45b0f1c26285417b900b3c5735721f7e
1 file changed
+68
-28
fast-import.c
+68
-28
@@ -217,6 +217,8 @@ struct hash_list
217
218
/* Stats and misc. counters */
219
static unsigned long max_depth = 10;
220
+static unsigned long max_objects = -1;
221
+static unsigned long max_packsize = -1;
222
static unsigned long alloc_count;
223
static unsigned long branch_count;
224
static unsigned long branch_load_count;
@@ -303,7 +305,6 @@ static struct object_entry* new_object(unsigned char *sha1)
305
alloc_objects(object_entry_alloc);
306
307
e = blocks->next_free++;
306
- e->pack_id = pack_id;
308
hashcpy(e->sha1, sha1);
309
return e;
310
}
@@ -678,12 +679,9 @@ static void write_index(const char *idx_name)
679
idx = xmalloc(object_count * sizeof(struct object_entry*));
680
c = idx;
681
for (o = blocks; o; o = o->next_pool)
681
- for (e = o->next_free; e-- != o->entries;) {
682
- if (pack_id != e->pack_id)
683
- goto sort_index;
684
- *c++ = e;
685
- }
686
-sort_index:
682
+ for (e = o->next_free; e-- != o->entries;)
683
+ if (pack_id == e->pack_id)
684
+ *c++ = e;
685
last = idx + object_count;
686
if (c != last)
687
die("internal consistency error creating the index");
@@ -746,6 +744,12 @@ static void end_packfile()
744
last_blob.depth = 0;
745
}
746
747
+static void checkpoint()
748
+{
749
+ end_packfile();
750
+ start_packfile();
751
+}
752
+
753
static size_t encode_header(
754
enum object_type type,
755
size_t size,
@@ -800,20 +804,64 @@ static int store_object(
804
duplicate_count_by_type[type]++;
805
return 1;
806
}
803
- e->type = type;
804
- e->offset = pack_size;
805
- object_count++;
806
- object_count_by_type[type]++;
807
808
- if (last && last->data && last->depth < max_depth)
808
+ if (last && last->data && last->depth < max_depth) {
809
delta = diff_delta(last->data, last->len,
810
dat, datlen,
811
&deltalen, 0);
812
- else
813
- delta = 0;
812
+ if (delta && deltalen >= datlen) {
813
+ free(delta);
814
+ delta = NULL;
815
+ }
816
+ } else
817
+ delta = NULL;
818
819
memset(&s, 0, sizeof(s));
820
deflateInit(&s, zlib_compression_level);
821
+ if (delta) {
822
+ s.next_in = delta;
823
+ s.avail_in = deltalen;
824
+ } else {
825
+ s.next_in = dat;
826
+ s.avail_in = datlen;
827
+ }
828
+ s.avail_out = deflateBound(&s, s.avail_in);
829
+ s.next_out = out = xmalloc(s.avail_out);
830
+ while (deflate(&s, Z_FINISH) == Z_OK)
831
+ /* nothing */;
832
+ deflateEnd(&s);
833
+
834
+ /* Determine if we should auto-checkpoint. */
835
+ if ((object_count + 1) > max_objects
836
+ || (object_count + 1) < object_count
837
+ || (pack_size + 60 + s.total_out) > max_packsize
838
+ || (pack_size + 60 + s.total_out) < pack_size) {
839
+
840
+ /* This new object needs to *not* have the current pack_id. */
841
+ e->pack_id = pack_id + 1;
842
+ checkpoint();
843
+
844
+ /* We cannot carry a delta into the new pack. */
845
+ if (delta) {
846
+ free(delta);
847
+ delta = NULL;
848
+ }
849
+ memset(&s, 0, sizeof(s));
850
+ deflateInit(&s, zlib_compression_level);
851
+ s.next_in = dat;
852
+ s.avail_in = datlen;
853
+ s.avail_out = deflateBound(&s, s.avail_in);
854
+ s.next_out = out;
855
+ while (deflate(&s, Z_FINISH) == Z_OK)
856
+ /* nothing */;
857
+ deflateEnd(&s);
858
+ }
859
+
860
+ e->type = type;
861
+ e->pack_id = pack_id;
862
+ e->offset = pack_size;
863
+ object_count++;
864
+ object_count_by_type[type]++;
865
866
if (delta) {
867
unsigned long ofs = e->offset - last->offset;
@@ -821,8 +869,6 @@ static int store_object(
869
870
delta_count_by_type[type]++;
871
last->depth++;
824
- s.next_in = delta;
825
- s.avail_in = deltalen;
872
873
hdrlen = encode_header(OBJ_OFS_DELTA, deltalen, hdr);
874
write_or_die(pack_fd, hdr, hdrlen);
@@ -836,19 +882,11 @@ static int store_object(
882
} else {
883
if (last)
884
last->depth = 0;
839
- s.next_in = dat;
840
- s.avail_in = datlen;
885
hdrlen = encode_header(type, datlen, hdr);
886
write_or_die(pack_fd, hdr, hdrlen);
887
pack_size += hdrlen;
888
}
889
846
- s.avail_out = deflateBound(&s, s.avail_in);
847
- s.next_out = out = xmalloc(s.avail_out);
848
- while (deflate(&s, Z_FINISH) == Z_OK)
849
- /* nothing */;
850
- deflateEnd(&s);
851
-
890
write_or_die(pack_fd, out, s.total_out);
891
pack_size += s.total_out;
892
@@ -1754,10 +1792,8 @@ static void cmd_reset_branch()
1792
1793
static void cmd_checkpoint()
1794
{
1757
- if (object_count) {
1758
- end_packfile();
1759
- start_packfile();
1760
- }
1795
+ if (object_count)
1796
+ checkpoint();
1797
read_next_command();
1798
}
1799
@@ -1780,6 +1816,10 @@ int main(int argc, const char **argv)
1816
break;
1817
else if (!strncmp(a, "--objects=", 10))
1818
est_obj_cnt = strtoul(a + 10, NULL, 0);
1819
+ else if (!strncmp(a, "--max-objects-per-pack=", 23))
1820
+ max_objects = strtoul(a + 23, NULL, 0);
1821
+ else if (!strncmp(a, "--max-pack-size=", 16))
1822
+ max_packsize = strtoul(a + 16, NULL, 0) * 1024 * 1024;
1823
else if (!strncmp(a, "--depth=", 8))
1824
max_depth = strtoul(a + 8, NULL, 0);
1825
else if (!strncmp(a, "--active-branches=", 18))