@matheusbsbs / matheus-bagliano / commits / 94c3b48247

fast-import: count and report # of calls to diff_delta in stats

It's an interesting number, how often do we try to deltify each type of objects and how often do we succeed. So do add it to stats. Success doesn't mean much gain in pack size though. As we allow delta to be as big as (data.len - 20). And delta close to data.len gains nothing compared to no delta at all even after zlib compression (delta is pretty much the same as data, just with few modifications). We should try to make less attempts that result in huge deltas as these consume more cpu than trivial small deltas. Either by choosing a better delta base or reducing delta size upper bound or doing less delta attempts at all. Currently, delta base for blobs is a waste literally. Each blob delta base is chosen as a previously stored blob. Disabling deltas for blobs doesn't increase pack size and reduce import time, or at least doesn't increase time for all fast-import streams I've tried. Signed-off-by: Dmitry Ivankov <divanorama@gmail.com> Acked-by: David Barr <davidbarr@google.com> Acked-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Dmitry Ivankov committed Aug 21, 2011 at 01:04 UTC 94c3b48247b25773d9c0d7de892cb75cb06708bb

1 file changed +6 -4

fast-import.c

+6 -4

@@ -284,6 +284,7 @@ static uintmax_t marks_set_count;

284 static uintmax_t object_count_by_type[1 << TYPE_BITS];

285 static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];

286 static uintmax_t delta_count_by_type[1 << TYPE_BITS];

287 +static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS];

288 static unsigned long object_count;

289 static unsigned long branch_count;

290 static unsigned long branch_load_count;

@@ -1043,6 +1044,7 @@ static int store_object(

1044 }

1045

1046 if (last && last->data.buf && last->depth < max_depth && dat->len > 20) {

1047 + delta_count_attempts_by_type[type]++;

1048 delta = diff_delta(last->data.buf, last->data.len,

1049 dat->buf, dat->len,

1050 &deltalen, dat->len - 20);

@@ -3328,10 +3330,10 @@ int main(int argc, const char **argv)

3330 fprintf(stderr, "---------------------------------------------------------------------\n");

3331 fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count);

3332 fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count);

3331 - fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);

3332 - fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);

3333 - fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);

3334 - fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);

3333 + fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]);

3334 + fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]);

3335 + fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]);

3336 + fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]);

3337 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);

3338 fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);

3339 fprintf(stderr, " atoms: %10u\n", atom_cnt);