pack-objects: shrink delta_size field in struct object_entry
Allowing a delta size of 64 bits is crazy. Shrink this field down to 20 bits with one overflow bit. If we find an existing delta larger than 1MB, we do not cache delta_size at all and will get the value from oe_size(), potentially from disk if it's larger than 4GB. Note, since DELTA_SIZE() is used in try_delta() code, it must be thread-safe. Luckily oe_size() does guarantee this so we it is thread-safe. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Apr 14, 2018 at 17:35 UTC
0aca34e8269514ebb67676e0470a314615494ae8
2 files changed
+38
-11
builtin/pack-objects.c
+16
-10
@@ -34,10 +34,12 @@
34
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
35
#define SIZE(obj) oe_size(&to_pack, obj)
36
#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size)
37
+#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj)
38
#define DELTA(obj) oe_delta(&to_pack, obj)
39
#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
40
#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
41
#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
42
+#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
43
#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
44
#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
45
@@ -144,7 +146,7 @@ static void *get_delta(struct object_entry *entry)
146
oid_to_hex(&DELTA(entry)->idx.oid));
147
delta_buf = diff_delta(base_buf, base_size,
148
buf, size, &delta_size, 0);
147
- if (!delta_buf || delta_size != entry->delta_size)
149
+ if (!delta_buf || delta_size != DELTA_SIZE(entry))
150
die("delta size changed");
151
free(buf);
152
free(base_buf);
@@ -294,14 +296,14 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
296
FREE_AND_NULL(entry->delta_data);
297
entry->z_delta_size = 0;
298
} else if (entry->delta_data) {
297
- size = entry->delta_size;
299
+ size = DELTA_SIZE(entry);
300
buf = entry->delta_data;
301
entry->delta_data = NULL;
302
type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
303
OBJ_OFS_DELTA : OBJ_REF_DELTA;
304
} else {
305
buf = get_delta(entry);
304
- size = entry->delta_size;
306
+ size = DELTA_SIZE(entry);
307
type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
308
OBJ_OFS_DELTA : OBJ_REF_DELTA;
309
}
@@ -1509,7 +1511,7 @@ static void check_object(struct object_entry *entry)
1511
oe_set_type(entry, entry->in_pack_type);
1512
SET_SIZE(entry, in_pack_size); /* delta size */
1513
SET_DELTA(entry, base_entry);
1512
- entry->delta_size = in_pack_size;
1514
+ SET_DELTA_SIZE(entry, in_pack_size);
1515
entry->delta_sibling_idx = base_entry->delta_child_idx;
1516
SET_DELTA_CHILD(base_entry, entry);
1517
unuse_pack(&w_curs);
@@ -1937,7 +1939,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1939
max_size = trg_size/2 - 20;
1940
ref_depth = 1;
1941
} else {
1940
- max_size = trg_entry->delta_size;
1942
+ max_size = DELTA_SIZE(trg_entry);
1943
ref_depth = trg->depth;
1944
}
1945
max_size = (uint64_t)max_size * (max_depth - src->depth) /
@@ -2006,10 +2008,14 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
2008
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
2009
if (!delta_buf)
2010
return 0;
2011
+ if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) {
2012
+ free(delta_buf);
2013
+ return 0;
2014
+ }
2015
2016
if (DELTA(trg_entry)) {
2017
/* Prefer only shallower same-sized deltas. */
2012
- if (delta_size == trg_entry->delta_size &&
2018
+ if (delta_size == DELTA_SIZE(trg_entry) &&
2019
src->depth + 1 >= trg->depth) {
2020
free(delta_buf);
2021
return 0;
@@ -2024,7 +2030,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
2030
free(trg_entry->delta_data);
2031
cache_lock();
2032
if (trg_entry->delta_data) {
2027
- delta_cache_size -= trg_entry->delta_size;
2033
+ delta_cache_size -= DELTA_SIZE(trg_entry);
2034
trg_entry->delta_data = NULL;
2035
}
2036
if (delta_cacheable(src_size, trg_size, delta_size)) {
@@ -2037,7 +2043,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
2043
}
2044
2045
SET_DELTA(trg_entry, src_entry);
2040
- trg_entry->delta_size = delta_size;
2046
+ SET_DELTA_SIZE(trg_entry, delta_size);
2047
trg->depth = src->depth + 1;
2048
2049
return 1;
@@ -2160,11 +2166,11 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
2166
if (entry->delta_data && !pack_to_stdout) {
2167
unsigned long size;
2168
2163
- size = do_compress(&entry->delta_data, entry->delta_size);
2169
+ size = do_compress(&entry->delta_data, DELTA_SIZE(entry));
2170
if (size < (1U << OE_Z_DELTA_BITS)) {
2171
entry->z_delta_size = size;
2172
cache_lock();
2167
- delta_cache_size -= entry->delta_size;
2173
+ delta_cache_size -= DELTA_SIZE(entry);
2174
delta_cache_size += entry->z_delta_size;
2175
cache_unlock();
2176
} else {
pack-objects.h
+22
-1
@@ -12,6 +12,7 @@
12
* above this limit. Don't lower it too much.
13
*/
14
#define OE_SIZE_BITS 31
15
+#define OE_DELTA_SIZE_BITS 20
16
17
/*
18
* State flags for depth-first search used for analyzing delta cycles.
@@ -85,7 +86,8 @@ struct object_entry {
86
* uses the same base as me
87
*/
88
void *delta_data; /* cached delta (uncompressed) */
88
- unsigned long delta_size; /* delta data size (uncompressed) */
89
+ unsigned delta_size_:OE_DELTA_SIZE_BITS; /* delta data size (uncompressed) */
90
+ unsigned delta_size_valid:1;
91
unsigned z_delta_size:OE_Z_DELTA_BITS;
92
unsigned type_:TYPE_BITS;
93
unsigned in_pack_type:TYPE_BITS; /* could be delta */
@@ -309,4 +311,23 @@ static inline void oe_set_size(struct packing_data *pack,
311
}
312
}
313
314
+static inline unsigned long oe_delta_size(struct packing_data *pack,
315
+ const struct object_entry *e)
316
+{
317
+ if (e->delta_size_valid)
318
+ return e->delta_size_;
319
+ return oe_size(pack, e);
320
+}
321
+
322
+static inline void oe_set_delta_size(struct packing_data *pack,
323
+ struct object_entry *e,
324
+ unsigned long size)
325
+{
326
+ e->delta_size_ = size;
327
+ e->delta_size_valid = e->delta_size_ == size;
328
+ if (!e->delta_size_valid && size != oe_size(pack, e))
329
+ BUG("this can only happen in check_object() "
330
+ "where delta size is the same as entry size");
331
+}
332
+
333
#endif