@matheusbsbs / matheus-bagliano / commits / ed4a9031ea

Fork 0

pack-objects: allow for early delta deflating

When the delta data is cached in memory until it is written to a pack file on disk, it is best to compress it right away in find_deltas() for the following reasons: - we have to compress that data anyway; - this allows for caching more deltas with the same cache size limit; - compression is potentially threaded. This last point is especially relevant for SMP run time. For example, repacking the Linux repo on a quad core processor using 4 threads with all default settings produce the following results before this change: real 2m27.929s user 4m36.492s sys 0m3.091s And with this change applied: real 2m13.787s user 4m37.486s sys 0m3.159s So the actual execution time stayed more or less the same but the wall clock time is shorter. This is however not a good thing to do when generating a pack for network transmission. In that case, the network is most likely to throttle the data throughput, so it is best to make find_deltas() faster in order to start writing data ASAP since we can afford spending more time between writes to compress the data at that point. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nicolas Pitre committed May 2, 2008 at 15:11 UTC ed4a9031ea75858d6f8ec387993e8ff731e29d1a

1 file changed +36 -1

builtin-pack-objects.c

+36 -1

@@ -43,6 +43,7 @@ struct object_entry {

43 */

44 void *delta_data; /* cached delta (uncompressed) */

45 unsigned long delta_size; /* delta data size (uncompressed) */

46 + unsigned long z_delta_size; /* delta data size (compressed) */

47 unsigned int hash; /* name hint hash */

48 enum object_type type;

49 enum object_type in_pack_type; /* could be delta */

@@ -301,6 +302,13 @@ static unsigned long write_object(struct sha1file *f,

302 buf = read_sha1_file(entry->idx.sha1, &type, &size);

303 if (!buf)

304 die("unable to read %s", sha1_to_hex(entry->idx.sha1));

305 + /*

306 + * make sure no cached delta data remains from a

307 + * previous attempt before a pack split occured.

308 + */

309 + free(entry->delta_data);

310 + entry->delta_data = NULL;

311 + entry->z_delta_size = 0;

312 } else if (entry->delta_data) {

313 size = entry->delta_size;

314 buf = entry->delta_data;

@@ -313,7 +321,11 @@ static unsigned long write_object(struct sha1file *f,

321 type = (allow_ofs_delta && entry->delta->idx.offset) ?

322 OBJ_OFS_DELTA : OBJ_REF_DELTA;

323 }

316 - datalen = do_compress(&buf, size);

324 +

325 + if (entry->z_delta_size)

326 + datalen = entry->z_delta_size;

327 + else

328 + datalen = do_compress(&buf, size);

329

330 /*

331 * The object header is a byte of 'type' followed by zero or

@@ -1447,6 +1459,29 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,

1459 best_base = other_idx;

1460 }

1461

1462 + /*

1463 + * If we decided to cache the delta data, then it is best

1464 + * to compress it right away. First because we have to do

1465 + * it anyway, and doing it here while we're threaded will

1466 + * save a lot of time in the non threaded write phase,

1467 + * as well as allow for caching more deltas within

1468 + * the same cache size limit.

1469 + * ...

1470 + * But only if not writing to stdout, since in that case

1471 + * the network is most likely throttling writes anyway,

1472 + * and therefore it is best to go to the write phase ASAP

1473 + * instead, as we can afford spending more time compressing

1474 + * between writes at that moment.

1475 + */

1476 + if (entry->delta_data && !pack_to_stdout) {

1477 + entry->z_delta_size = do_compress(&entry->delta_data,

1478 + entry->delta_size);

1479 + cache_lock();

1480 + delta_cache_size -= entry->delta_size;

1481 + delta_cache_size += entry->z_delta_size;

1482 + cache_unlock();

1483 + }

1484 +

1485 /* if we made n a delta, and if n is already at max

1486 * depth, leaving it in the window is pointless. we

1487 * should evict it first.