@matheusbsbs / matheus-bagliano / commits / d9ee53ce45
Fork 0

Implemented automatic checkpoints within fast-import.

When the number of objects or number of bytes gets close to the limit allowed by the packfile format (or configured on the command line by our caller) we should automatically checkpoint the current packfile and start a new one before writing the object out. This does however require that we abandon the delta (if we had one) as its not valid in a new packfile. I also added the simple rule that if we got a delta back but the delta itself is the same size as or larger than the uncompressed object to ignore the delta and just store the object data. This should avoid some really bad behavior caused by our current delta strategy. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Jan 15, 2007 at 08:00 UTC d9ee53ce45b0f1c26285417b900b3c5735721f7e
1 file changed +68 -28
fast-import.c
+68 -28
@@ -217,6 +217,8 @@ struct hash_list
217
218 /* Stats and misc. counters */
219 static unsigned long max_depth = 10;
220 +static unsigned long max_objects = -1;
221 +static unsigned long max_packsize = -1;
222 static unsigned long alloc_count;
223 static unsigned long branch_count;
224 static unsigned long branch_load_count;
@@ -303,7 +305,6 @@ static struct object_entry* new_object(unsigned char *sha1)
305 alloc_objects(object_entry_alloc);
306
307 e = blocks->next_free++;
306 - e->pack_id = pack_id;
308 hashcpy(e->sha1, sha1);
309 return e;
310 }
@@ -678,12 +679,9 @@ static void write_index(const char *idx_name)
679 idx = xmalloc(object_count * sizeof(struct object_entry*));
680 c = idx;
681 for (o = blocks; o; o = o->next_pool)
681 - for (e = o->next_free; e-- != o->entries;) {
682 - if (pack_id != e->pack_id)
683 - goto sort_index;
684 - *c++ = e;
685 - }
686 -sort_index:
682 + for (e = o->next_free; e-- != o->entries;)
683 + if (pack_id == e->pack_id)
684 + *c++ = e;
685 last = idx + object_count;
686 if (c != last)
687 die("internal consistency error creating the index");
@@ -746,6 +744,12 @@ static void end_packfile()
744 last_blob.depth = 0;
745 }
746
747 +static void checkpoint()
748 +{
749 + end_packfile();
750 + start_packfile();
751 +}
752 +
753 static size_t encode_header(
754 enum object_type type,
755 size_t size,
@@ -800,20 +804,64 @@ static int store_object(
804 duplicate_count_by_type[type]++;
805 return 1;
806 }
803 - e->type = type;
804 - e->offset = pack_size;
805 - object_count++;
806 - object_count_by_type[type]++;
807
808 - if (last && last->data && last->depth < max_depth)
808 + if (last && last->data && last->depth < max_depth) {
809 delta = diff_delta(last->data, last->len,
810 dat, datlen,
811 &deltalen, 0);
812 - else
813 - delta = 0;
812 + if (delta && deltalen >= datlen) {
813 + free(delta);
814 + delta = NULL;
815 + }
816 + } else
817 + delta = NULL;
818
819 memset(&s, 0, sizeof(s));
820 deflateInit(&s, zlib_compression_level);
821 + if (delta) {
822 + s.next_in = delta;
823 + s.avail_in = deltalen;
824 + } else {
825 + s.next_in = dat;
826 + s.avail_in = datlen;
827 + }
828 + s.avail_out = deflateBound(&s, s.avail_in);
829 + s.next_out = out = xmalloc(s.avail_out);
830 + while (deflate(&s, Z_FINISH) == Z_OK)
831 + /* nothing */;
832 + deflateEnd(&s);
833 +
834 + /* Determine if we should auto-checkpoint. */
835 + if ((object_count + 1) > max_objects
836 + || (object_count + 1) < object_count
837 + || (pack_size + 60 + s.total_out) > max_packsize
838 + || (pack_size + 60 + s.total_out) < pack_size) {
839 +
840 + /* This new object needs to *not* have the current pack_id. */
841 + e->pack_id = pack_id + 1;
842 + checkpoint();
843 +
844 + /* We cannot carry a delta into the new pack. */
845 + if (delta) {
846 + free(delta);
847 + delta = NULL;
848 + }
849 + memset(&s, 0, sizeof(s));
850 + deflateInit(&s, zlib_compression_level);
851 + s.next_in = dat;
852 + s.avail_in = datlen;
853 + s.avail_out = deflateBound(&s, s.avail_in);
854 + s.next_out = out;
855 + while (deflate(&s, Z_FINISH) == Z_OK)
856 + /* nothing */;
857 + deflateEnd(&s);
858 + }
859 +
860 + e->type = type;
861 + e->pack_id = pack_id;
862 + e->offset = pack_size;
863 + object_count++;
864 + object_count_by_type[type]++;
865
866 if (delta) {
867 unsigned long ofs = e->offset - last->offset;
@@ -821,8 +869,6 @@ static int store_object(
869
870 delta_count_by_type[type]++;
871 last->depth++;
824 - s.next_in = delta;
825 - s.avail_in = deltalen;
872
873 hdrlen = encode_header(OBJ_OFS_DELTA, deltalen, hdr);
874 write_or_die(pack_fd, hdr, hdrlen);
@@ -836,19 +882,11 @@ static int store_object(
882 } else {
883 if (last)
884 last->depth = 0;
839 - s.next_in = dat;
840 - s.avail_in = datlen;
885 hdrlen = encode_header(type, datlen, hdr);
886 write_or_die(pack_fd, hdr, hdrlen);
887 pack_size += hdrlen;
888 }
889
846 - s.avail_out = deflateBound(&s, s.avail_in);
847 - s.next_out = out = xmalloc(s.avail_out);
848 - while (deflate(&s, Z_FINISH) == Z_OK)
849 - /* nothing */;
850 - deflateEnd(&s);
851 -
890 write_or_die(pack_fd, out, s.total_out);
891 pack_size += s.total_out;
892
@@ -1754,10 +1792,8 @@ static void cmd_reset_branch()
1792
1793 static void cmd_checkpoint()
1794 {
1757 - if (object_count) {
1758 - end_packfile();
1759 - start_packfile();
1760 - }
1795 + if (object_count)
1796 + checkpoint();
1797 read_next_command();
1798 }
1799
@@ -1780,6 +1816,10 @@ int main(int argc, const char **argv)
1816 break;
1817 else if (!strncmp(a, "--objects=", 10))
1818 est_obj_cnt = strtoul(a + 10, NULL, 0);
1819 + else if (!strncmp(a, "--max-objects-per-pack=", 23))
1820 + max_objects = strtoul(a + 23, NULL, 0);
1821 + else if (!strncmp(a, "--max-pack-size=", 16))
1822 + max_packsize = strtoul(a + 16, NULL, 0) * 1024 * 1024;
1823 else if (!strncmp(a, "--depth=", 8))
1824 max_depth = strtoul(a + 8, NULL, 0);
1825 else if (!strncmp(a, "--active-branches=", 18))