pack-objects: refer to delta objects by index instead of pointer
These delta pointers always point to elements in the objects[] array in packing_data struct. We can only hold maximum 4G of those objects because the array size in nr_objects is uint32_t. We could use uint32_t indexes to address these elements instead of pointers. On 64-bit architecture (8 bytes per pointer) this would save 4 bytes per pointer. Convert these delta pointers to indexes. Since we need to handle NULL pointers as well, the index is shifted by one [1]. [1] This means we can only index 2^32-2 objects even though nr_objects could contain 2^32-1 objects. It should not be a problem in practice because when we grow objects[], nr_alloc would probably blow up long before nr_objects hits the wall. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Apr 14, 2018 at 17:35 UTC
898eba5e630696608ddca0ede489378e87464ad6
2 files changed
+125
-59
builtin/pack-objects.c
+63
-54
@@ -32,6 +32,12 @@
32
#include "object-store.h"
33
34
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
35
+#define DELTA(obj) oe_delta(&to_pack, obj)
36
+#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
37
+#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
38
+#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
39
+#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
40
+#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
41
42
static const char *pack_usage[] = {
43
N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"),
@@ -129,10 +135,11 @@ static void *get_delta(struct object_entry *entry)
135
buf = read_object_file(&entry->idx.oid, &type, &size);
136
if (!buf)
137
die("unable to read %s", oid_to_hex(&entry->idx.oid));
132
- base_buf = read_object_file(&entry->delta->idx.oid, &type, &base_size);
138
+ base_buf = read_object_file(&DELTA(entry)->idx.oid, &type,
139
+ &base_size);
140
if (!base_buf)
141
die("unable to read %s",
135
- oid_to_hex(&entry->delta->idx.oid));
142
+ oid_to_hex(&DELTA(entry)->idx.oid));
143
delta_buf = diff_delta(base_buf, base_size,
144
buf, size, &delta_size, 0);
145
if (!delta_buf || delta_size != entry->delta_size)
@@ -288,12 +295,12 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
295
size = entry->delta_size;
296
buf = entry->delta_data;
297
entry->delta_data = NULL;
291
- type = (allow_ofs_delta && entry->delta->idx.offset) ?
298
+ type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
299
OBJ_OFS_DELTA : OBJ_REF_DELTA;
300
} else {
301
buf = get_delta(entry);
302
size = entry->delta_size;
296
- type = (allow_ofs_delta && entry->delta->idx.offset) ?
303
+ type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
304
OBJ_OFS_DELTA : OBJ_REF_DELTA;
305
}
306
@@ -317,7 +324,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
324
* encoding of the relative offset for the delta
325
* base from this object's position in the pack.
326
*/
320
- off_t ofs = entry->idx.offset - entry->delta->idx.offset;
327
+ off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset;
328
unsigned pos = sizeof(dheader) - 1;
329
dheader[pos] = ofs & 127;
330
while (ofs >>= 7)
@@ -343,7 +350,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
350
return 0;
351
}
352
hashwrite(f, header, hdrlen);
346
- hashwrite(f, entry->delta->idx.oid.hash, 20);
353
+ hashwrite(f, DELTA(entry)->idx.oid.hash, 20);
354
hdrlen += 20;
355
} else {
356
if (limit && hdrlen + datalen + 20 >= limit) {
@@ -379,8 +386,8 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
386
dheader[MAX_PACK_OBJECT_HEADER];
387
unsigned hdrlen;
388
382
- if (entry->delta)
383
- type = (allow_ofs_delta && entry->delta->idx.offset) ?
389
+ if (DELTA(entry))
390
+ type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
391
OBJ_OFS_DELTA : OBJ_REF_DELTA;
392
hdrlen = encode_in_pack_object_header(header, sizeof(header),
393
type, entry->size);
@@ -408,7 +415,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
415
}
416
417
if (type == OBJ_OFS_DELTA) {
411
- off_t ofs = entry->idx.offset - entry->delta->idx.offset;
418
+ off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset;
419
unsigned pos = sizeof(dheader) - 1;
420
dheader[pos] = ofs & 127;
421
while (ofs >>= 7)
@@ -427,7 +434,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
434
return 0;
435
}
436
hashwrite(f, header, hdrlen);
430
- hashwrite(f, entry->delta->idx.oid.hash, 20);
437
+ hashwrite(f, DELTA(entry)->idx.oid.hash, 20);
438
hdrlen += 20;
439
reused_delta++;
440
} else {
@@ -467,13 +474,13 @@ static off_t write_object(struct hashfile *f,
474
else
475
limit = pack_size_limit - write_offset;
476
470
- if (!entry->delta)
477
+ if (!DELTA(entry))
478
usable_delta = 0; /* no delta */
479
else if (!pack_size_limit)
480
usable_delta = 1; /* unlimited packfile */
474
- else if (entry->delta->idx.offset == (off_t)-1)
481
+ else if (DELTA(entry)->idx.offset == (off_t)-1)
482
usable_delta = 0; /* base was written to another pack */
476
- else if (entry->delta->idx.offset)
483
+ else if (DELTA(entry)->idx.offset)
484
usable_delta = 1; /* base already exists in this pack */
485
else
486
usable_delta = 0; /* base could end up in another pack */
@@ -489,7 +496,7 @@ static off_t write_object(struct hashfile *f,
496
/* ... but pack split may override that */
497
else if (oe_type(entry) != entry->in_pack_type)
498
to_reuse = 0; /* pack has delta which is unusable */
492
- else if (entry->delta)
499
+ else if (DELTA(entry))
500
to_reuse = 0; /* we want to pack afresh */
501
else
502
to_reuse = 1; /* we have it in-pack undeltified,
@@ -541,12 +548,12 @@ static enum write_one_status write_one(struct hashfile *f,
548
}
549
550
/* if we are deltified, write out base object first. */
544
- if (e->delta) {
551
+ if (DELTA(e)) {
552
e->idx.offset = 1; /* now recurse */
546
- switch (write_one(f, e->delta, offset)) {
553
+ switch (write_one(f, DELTA(e), offset)) {
554
case WRITE_ONE_RECURSIVE:
555
/* we cannot depend on this one */
549
- e->delta = NULL;
556
+ SET_DELTA(e, NULL);
557
break;
558
default:
559
break;
@@ -608,34 +615,34 @@ static void add_descendants_to_write_order(struct object_entry **wo,
615
/* add this node... */
616
add_to_write_order(wo, endp, e);
617
/* all its siblings... */
611
- for (s = e->delta_sibling; s; s = s->delta_sibling) {
618
+ for (s = DELTA_SIBLING(e); s; s = DELTA_SIBLING(s)) {
619
add_to_write_order(wo, endp, s);
620
}
621
}
622
/* drop down a level to add left subtree nodes if possible */
616
- if (e->delta_child) {
623
+ if (DELTA_CHILD(e)) {
624
add_to_order = 1;
618
- e = e->delta_child;
625
+ e = DELTA_CHILD(e);
626
} else {
627
add_to_order = 0;
628
/* our sibling might have some children, it is next */
622
- if (e->delta_sibling) {
623
- e = e->delta_sibling;
629
+ if (DELTA_SIBLING(e)) {
630
+ e = DELTA_SIBLING(e);
631
continue;
632
}
633
/* go back to our parent node */
627
- e = e->delta;
628
- while (e && !e->delta_sibling) {
634
+ e = DELTA(e);
635
+ while (e && !DELTA_SIBLING(e)) {
636
/* we're on the right side of a subtree, keep
637
* going up until we can go right again */
631
- e = e->delta;
638
+ e = DELTA(e);
639
}
640
if (!e) {
641
/* done- we hit our original root node */
642
return;
643
}
644
/* pass it off to sibling at this level */
638
- e = e->delta_sibling;
645
+ e = DELTA_SIBLING(e);
646
}
647
};
648
}
@@ -646,7 +653,7 @@ static void add_family_to_write_order(struct object_entry **wo,
653
{
654
struct object_entry *root;
655
649
- for (root = e; root->delta; root = root->delta)
656
+ for (root = e; DELTA(root); root = DELTA(root))
657
; /* nothing */
658
add_descendants_to_write_order(wo, endp, root);
659
}
@@ -661,8 +668,8 @@ static struct object_entry **compute_write_order(void)
668
for (i = 0; i < to_pack.nr_objects; i++) {
669
objects[i].tagged = 0;
670
objects[i].filled = 0;
664
- objects[i].delta_child = NULL;
665
- objects[i].delta_sibling = NULL;
671
+ SET_DELTA_CHILD(&objects[i], NULL);
672
+ SET_DELTA_SIBLING(&objects[i], NULL);
673
}
674
675
/*
@@ -672,11 +679,11 @@ static struct object_entry **compute_write_order(void)
679
*/
680
for (i = to_pack.nr_objects; i > 0;) {
681
struct object_entry *e = &objects[--i];
675
- if (!e->delta)
682
+ if (!DELTA(e))
683
continue;
684
/* Mark me as the first child */
678
- e->delta_sibling = e->delta->delta_child;
679
- e->delta->delta_child = e;
685
+ e->delta_sibling_idx = DELTA(e)->delta_child_idx;
686
+ SET_DELTA_CHILD(DELTA(e), e);
687
}
688
689
/*
@@ -1493,10 +1500,10 @@ static void check_object(struct object_entry *entry)
1500
* circular deltas.
1501
*/
1502
oe_set_type(entry, entry->in_pack_type);
1496
- entry->delta = base_entry;
1503
+ SET_DELTA(entry, base_entry);
1504
entry->delta_size = entry->size;
1498
- entry->delta_sibling = base_entry->delta_child;
1499
- base_entry->delta_child = entry;
1505
+ entry->delta_sibling_idx = base_entry->delta_child_idx;
1506
+ SET_DELTA_CHILD(base_entry, entry);
1507
unuse_pack(&w_curs);
1508
return;
1509
}
@@ -1567,17 +1574,19 @@ static int pack_offset_sort(const void *_a, const void *_b)
1574
*/
1575
static void drop_reused_delta(struct object_entry *entry)
1576
{
1570
- struct object_entry **p = &entry->delta->delta_child;
1577
+ unsigned *idx = &to_pack.objects[entry->delta_idx - 1].delta_child_idx;
1578
struct object_info oi = OBJECT_INFO_INIT;
1579
enum object_type type;
1580
1574
- while (*p) {
1575
- if (*p == entry)
1576
- *p = (*p)->delta_sibling;
1581
+ while (*idx) {
1582
+ struct object_entry *oe = &to_pack.objects[*idx - 1];
1583
+
1584
+ if (oe == entry)
1585
+ *idx = oe->delta_sibling_idx;
1586
else
1578
- p = &(*p)->delta_sibling;
1587
+ idx = &oe->delta_sibling_idx;
1588
}
1580
- entry->delta = NULL;
1589
+ SET_DELTA(entry, NULL);
1590
entry->depth = 0;
1591
1592
oi.sizep = &entry->size;
@@ -1617,7 +1626,7 @@ static void break_delta_chains(struct object_entry *entry)
1626
1627
for (cur = entry, total_depth = 0;
1628
cur;
1620
- cur = cur->delta, total_depth++) {
1629
+ cur = DELTA(cur), total_depth++) {
1630
if (cur->dfs_state == DFS_DONE) {
1631
/*
1632
* We've already seen this object and know it isn't
@@ -1642,7 +1651,7 @@ static void break_delta_chains(struct object_entry *entry)
1651
* it's not a delta, we're done traversing, but we'll mark it
1652
* done to save time on future traversals.
1653
*/
1645
- if (!cur->delta) {
1654
+ if (!DELTA(cur)) {
1655
cur->dfs_state = DFS_DONE;
1656
break;
1657
}
@@ -1665,7 +1674,7 @@ static void break_delta_chains(struct object_entry *entry)
1674
* We keep all commits in the chain that we examined.
1675
*/
1676
cur->dfs_state = DFS_ACTIVE;
1668
- if (cur->delta->dfs_state == DFS_ACTIVE) {
1677
+ if (DELTA(cur)->dfs_state == DFS_ACTIVE) {
1678
drop_reused_delta(cur);
1679
cur->dfs_state = DFS_DONE;
1680
break;
@@ -1680,7 +1689,7 @@ static void break_delta_chains(struct object_entry *entry)
1689
* an extra "next" pointer to keep going after we reset cur->delta.
1690
*/
1691
for (cur = entry; cur; cur = next) {
1683
- next = cur->delta;
1692
+ next = DELTA(cur);
1693
1694
/*
1695
* We should have a chain of zero or more ACTIVE states down to
@@ -1865,7 +1874,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1874
1875
/* Now some size filtering heuristics. */
1876
trg_size = trg_entry->size;
1868
- if (!trg_entry->delta) {
1877
+ if (!DELTA(trg_entry)) {
1878
max_size = trg_size/2 - 20;
1879
ref_depth = 1;
1880
} else {
@@ -1939,7 +1948,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1948
if (!delta_buf)
1949
return 0;
1950
1942
- if (trg_entry->delta) {
1951
+ if (DELTA(trg_entry)) {
1952
/* Prefer only shallower same-sized deltas. */
1953
if (delta_size == trg_entry->delta_size &&
1954
src->depth + 1 >= trg->depth) {
@@ -1968,7 +1977,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1977
free(delta_buf);
1978
}
1979
1971
- trg_entry->delta = src_entry;
1980
+ SET_DELTA(trg_entry, src_entry);
1981
trg_entry->delta_size = delta_size;
1982
trg->depth = src->depth + 1;
1983
@@ -1977,13 +1986,13 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1986
1987
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
1988
{
1980
- struct object_entry *child = me->delta_child;
1989
+ struct object_entry *child = DELTA_CHILD(me);
1990
unsigned int m = n;
1991
while (child) {
1992
unsigned int c = check_delta_limit(child, n + 1);
1993
if (m < c)
1994
m = c;
1986
- child = child->delta_sibling;
1995
+ child = DELTA_SIBLING(child);
1996
}
1997
return m;
1998
}
@@ -2052,7 +2061,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
2061
* otherwise they would become too deep.
2062
*/
2063
max_depth = depth;
2055
- if (entry->delta_child) {
2064
+ if (DELTA_CHILD(entry)) {
2065
max_depth -= check_delta_limit(entry, 0);
2066
if (max_depth <= 0)
2067
goto next;
@@ -2102,7 +2111,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
2111
* depth, leaving it in the window is pointless. we
2112
* should evict it first.
2113
*/
2105
- if (entry->delta && max_depth <= n->depth)
2114
+ if (DELTA(entry) && max_depth <= n->depth)
2115
continue;
2116
2117
/*
@@ -2110,7 +2119,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
2119
* currently deltified object, to keep it longer. It will
2120
* be the first base object to be attempted next.
2121
*/
2113
- if (entry->delta) {
2122
+ if (DELTA(entry)) {
2123
struct unpacked swap = array[best_base];
2124
int dist = (window + idx - best_base) % window;
2125
int dst = best_base;
@@ -2431,7 +2440,7 @@ static void prepare_pack(int window, int depth)
2440
for (i = 0; i < to_pack.nr_objects; i++) {
2441
struct object_entry *entry = to_pack.objects + i;
2442
2434
- if (entry->delta)
2443
+ if (DELTA(entry))
2444
/* This happens if we decided to reuse existing
2445
* delta from a pack. "reuse_delta &&" is implied.
2446
*/
pack-objects.h
+62
-5
@@ -70,11 +70,11 @@ struct object_entry {
70
unsigned long size; /* uncompressed size */
71
unsigned in_pack_idx:OE_IN_PACK_BITS; /* already in pack */
72
off_t in_pack_offset;
73
- struct object_entry *delta; /* delta base object */
74
- struct object_entry *delta_child; /* deltified objects who bases me */
75
- struct object_entry *delta_sibling; /* other deltified objects who
76
- * uses the same base as me
77
- */
73
+ uint32_t delta_idx; /* delta base object */
74
+ uint32_t delta_child_idx; /* deltified objects who bases me */
75
+ uint32_t delta_sibling_idx; /* other deltified objects who
76
+ * uses the same base as me
77
+ */
78
void *delta_data; /* cached delta (uncompressed) */
79
unsigned long delta_size; /* delta data size (uncompressed) */
80
unsigned long z_delta_size; /* delta data size (compressed) */
@@ -194,4 +194,61 @@ static inline void oe_set_in_pack(struct packing_data *pack,
194
pack->in_pack[e - pack->objects] = p;
195
}
196
197
+static inline struct object_entry *oe_delta(
198
+ const struct packing_data *pack,
199
+ const struct object_entry *e)
200
+{
201
+ if (e->delta_idx)
202
+ return &pack->objects[e->delta_idx - 1];
203
+ return NULL;
204
+}
205
+
206
+static inline void oe_set_delta(struct packing_data *pack,
207
+ struct object_entry *e,
208
+ struct object_entry *delta)
209
+{
210
+ if (delta)
211
+ e->delta_idx = (delta - pack->objects) + 1;
212
+ else
213
+ e->delta_idx = 0;
214
+}
215
+
216
+static inline struct object_entry *oe_delta_child(
217
+ const struct packing_data *pack,
218
+ const struct object_entry *e)
219
+{
220
+ if (e->delta_child_idx)
221
+ return &pack->objects[e->delta_child_idx - 1];
222
+ return NULL;
223
+}
224
+
225
+static inline void oe_set_delta_child(struct packing_data *pack,
226
+ struct object_entry *e,
227
+ struct object_entry *delta)
228
+{
229
+ if (delta)
230
+ e->delta_child_idx = (delta - pack->objects) + 1;
231
+ else
232
+ e->delta_child_idx = 0;
233
+}
234
+
235
+static inline struct object_entry *oe_delta_sibling(
236
+ const struct packing_data *pack,
237
+ const struct object_entry *e)
238
+{
239
+ if (e->delta_sibling_idx)
240
+ return &pack->objects[e->delta_sibling_idx - 1];
241
+ return NULL;
242
+}
243
+
244
+static inline void oe_set_delta_sibling(struct packing_data *pack,
245
+ struct object_entry *e,
246
+ struct object_entry *delta)
247
+{
248
+ if (delta)
249
+ e->delta_sibling_idx = (delta - pack->objects) + 1;
250
+ else
251
+ e->delta_sibling_idx = 0;
252
+}
253
+
254
#endif