git-pack-objects: cache small deltas between big objects
Creating deltas between big blobs is a CPU and memory intensive task. In the writing phase, all (not reused) deltas are redone. This patch adds support for caching deltas from the deltifing phase, so that that the writing phase is faster. The caching is limited to small deltas to avoid increasing memory usage very much. The implemented limit is (memory needed to create the delta)/1024. Signed-off-by: Martin Koegler <mkoegler@auto.tuwien.ac.at> Signed-off-by: Junio C Hamano <junkio@cox.net>
Martin Koegler committed
May 28, 2007 at 23:20 UTC
074b2eea296886e179ef73e1c364f370a223618a
2 files changed
+59
-15
Documentation/config.txt
+5
@@ -567,6 +567,11 @@ pack.compression::
567
slowest. If not set, defaults to core.compression. If that is
568
not set, defaults to -1.
569
570
+pack.deltaCacheSize::
571
+ The maxium memory in bytes used for caching deltas in
572
+ gitlink:git-pack-objects[1].
573
+ A value of 0 means no limit. Defaults to 0.
574
+
575
pull.octopus::
576
The default merge strategy to use when pulling multiple branches
577
at once.
builtin-pack-objects.c
+54
-15
@@ -36,6 +36,7 @@ struct object_entry {
36
struct object_entry *delta_sibling; /* other deltified objects who
37
* uses the same base as me
38
*/
39
+ void *delta_data; /* cached delta (uncompressed) */
40
unsigned long delta_size; /* delta data size (uncompressed) */
41
enum object_type type;
42
enum object_type in_pack_type; /* could be delta */
@@ -76,6 +77,9 @@ static struct progress progress_state;
77
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
78
static int pack_compression_seen;
79
80
+static unsigned long delta_cache_size = 0;
81
+static unsigned long max_delta_cache_size = 0;
82
+
83
/*
84
* The object names in objects array are hashed with this hashtable,
85
* to help looking up the entry by object name.
@@ -405,24 +409,31 @@ static unsigned long write_object(struct sha1file *f,
409
z_stream stream;
410
unsigned long maxsize;
411
void *out;
408
- buf = read_sha1_file(entry->sha1, &type, &size);
409
- if (!buf)
410
- die("unable to read %s", sha1_to_hex(entry->sha1));
411
- if (size != entry->size)
412
- die("object %s size inconsistency (%lu vs %lu)",
413
- sha1_to_hex(entry->sha1), size, entry->size);
414
- if (usable_delta) {
415
- buf = delta_against(buf, size, entry);
412
+ if (entry->delta_data && usable_delta) {
413
+ buf = entry->delta_data;
414
size = entry->delta_size;
415
obj_type = (allow_ofs_delta && entry->delta->offset) ?
416
OBJ_OFS_DELTA : OBJ_REF_DELTA;
417
} else {
420
- /*
421
- * recover real object type in case
422
- * check_object() wanted to re-use a delta,
423
- * but we couldn't since base was in previous split pack
424
- */
425
- obj_type = type;
418
+ buf = read_sha1_file(entry->sha1, &type, &size);
419
+ if (!buf)
420
+ die("unable to read %s", sha1_to_hex(entry->sha1));
421
+ if (size != entry->size)
422
+ die("object %s size inconsistency (%lu vs %lu)",
423
+ sha1_to_hex(entry->sha1), size, entry->size);
424
+ if (usable_delta) {
425
+ buf = delta_against(buf, size, entry);
426
+ size = entry->delta_size;
427
+ obj_type = (allow_ofs_delta && entry->delta->offset) ?
428
+ OBJ_OFS_DELTA : OBJ_REF_DELTA;
429
+ } else {
430
+ /*
431
+ * recover real object type in case
432
+ * check_object() wanted to re-use a delta,
433
+ * but we couldn't since base was in previous split pack
434
+ */
435
+ obj_type = type;
436
+ }
437
}
438
/* compress the data to store and put compressed length in datalen */
439
memset(&stream, 0, sizeof(stream));
@@ -1385,6 +1396,20 @@ struct unpacked {
1396
struct delta_index *index;
1397
};
1398
1399
+static int delta_cacheable(struct unpacked *trg, struct unpacked *src,
1400
+ unsigned long src_size, unsigned long trg_size,
1401
+ unsigned long delta_size)
1402
+{
1403
+ if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
1404
+ return 0;
1405
+
1406
+ /* cache delta, if objects are large enough compared to delta size */
1407
+ if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
1408
+ return 1;
1409
+
1410
+ return 0;
1411
+}
1412
+
1413
/*
1414
* We search for deltas _backwards_ in a list sorted by type and
1415
* by size, so that we see progressively smaller and smaller files.
@@ -1466,10 +1491,20 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1491
if (!delta_buf)
1492
return 0;
1493
1494
+ if (trg_entry->delta_data) {
1495
+ delta_cache_size -= trg_entry->delta_size;
1496
+ free(trg_entry->delta_data);
1497
+ }
1498
+ trg_entry->delta_data = 0;
1499
trg_entry->delta = src_entry;
1500
trg_entry->delta_size = delta_size;
1501
trg_entry->depth = src_entry->depth + 1;
1472
- free(delta_buf);
1502
+
1503
+ if (delta_cacheable(src, trg, src_size, trg_size, delta_size)) {
1504
+ trg_entry->delta_data = xrealloc(delta_buf, delta_size);
1505
+ delta_cache_size += trg_entry->delta_size;
1506
+ } else
1507
+ free(delta_buf);
1508
return 1;
1509
}
1510
@@ -1615,6 +1650,10 @@ static int git_pack_config(const char *k, const char *v)
1650
pack_compression_seen = 1;
1651
return 0;
1652
}
1653
+ if (!strcmp(k, "pack.deltacachesize")) {
1654
+ max_delta_cache_size = git_config_int(k, v);
1655
+ return 0;
1656
+ }
1657
return git_default_config(k, v);
1658
}
1659