make pack data reuse compatible with both delta types
This is the missing part to git-pack-objects allowing it to reuse delta data to/from any of the two delta types. It can reuse delta from any type, and it outputs base offsets when --allow-delta-base-offset is provided and the base is also included in the pack. Otherwise it outputs base sha1 references just like it always did. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nicolas Pitre committed
Sep 22, 2006 at 21:25 UTC
780e6e735be189097dad4b223d8edeb18cce1928
3 files changed
+130
-96
builtin-pack-objects.c
+130
-74
@@ -29,6 +29,7 @@ struct object_entry {
29
enum object_type type;
30
enum object_type in_pack_type; /* could be delta */
31
unsigned long delta_size; /* delta data size (uncompressed) */
32
+#define in_pack_header_size delta_size /* only when reusing pack data */
33
struct object_entry *delta; /* delta base object */
34
struct packed_git *in_pack; /* already in pack */
35
unsigned int in_pack_offset;
@@ -86,17 +87,25 @@ static int object_ix_hashsz;
87
* Pack index for existing packs give us easy access to the offsets into
88
* corresponding pack file where each object's data starts, but the entries
89
* do not store the size of the compressed representation (uncompressed
89
- * size is easily available by examining the pack entry header). We build
90
- * a hashtable of existing packs (pack_revindex), and keep reverse index
91
- * here -- pack index file is sorted by object name mapping to offset; this
92
- * pack_revindex[].revindex array is an ordered list of offsets, so if you
93
- * know the offset of an object, next offset is where its packed
94
- * representation ends.
90
+ * size is easily available by examining the pack entry header). It is
91
+ * also rather expensive to find the sha1 for an object given its offset.
92
+ *
93
+ * We build a hashtable of existing packs (pack_revindex), and keep reverse
94
+ * index here -- pack index file is sorted by object name mapping to offset;
95
+ * this pack_revindex[].revindex array is a list of offset/index_nr pairs
96
+ * ordered by offset, so if you know the offset of an object, next offset
97
+ * is where its packed representation ends and the index_nr can be used to
98
+ * get the object sha1 from the main index.
99
*/
100
+struct revindex_entry {
101
+ unsigned int offset;
102
+ unsigned int nr;
103
+};
104
struct pack_revindex {
105
struct packed_git *p;
98
- unsigned long *revindex;
99
-} *pack_revindex = NULL;
106
+ struct revindex_entry *revindex;
107
+};
108
+static struct pack_revindex *pack_revindex;
109
static int pack_revindex_hashsz;
110
111
/*
@@ -143,14 +152,9 @@ static void prepare_pack_ix(void)
152
153
static int cmp_offset(const void *a_, const void *b_)
154
{
146
- unsigned long a = *(unsigned long *) a_;
147
- unsigned long b = *(unsigned long *) b_;
148
- if (a < b)
149
- return -1;
150
- else if (a == b)
151
- return 0;
152
- else
153
- return 1;
155
+ const struct revindex_entry *a = a_;
156
+ const struct revindex_entry *b = b_;
157
+ return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
158
}
159
160
/*
@@ -163,25 +167,27 @@ static void prepare_pack_revindex(struct pack_revindex *rix)
167
int i;
168
void *index = p->index_base + 256;
169
166
- rix->revindex = xmalloc(sizeof(unsigned long) * (num_ent + 1));
170
+ rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
171
for (i = 0; i < num_ent; i++) {
172
unsigned int hl = *((unsigned int *)((char *) index + 24*i));
169
- rix->revindex[i] = ntohl(hl);
173
+ rix->revindex[i].offset = ntohl(hl);
174
+ rix->revindex[i].nr = i;
175
}
176
/* This knows the pack format -- the 20-byte trailer
177
* follows immediately after the last object data.
178
*/
174
- rix->revindex[num_ent] = p->pack_size - 20;
175
- qsort(rix->revindex, num_ent, sizeof(unsigned long), cmp_offset);
179
+ rix->revindex[num_ent].offset = p->pack_size - 20;
180
+ rix->revindex[num_ent].nr = -1;
181
+ qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
182
}
183
178
-static unsigned long find_packed_object_size(struct packed_git *p,
179
- unsigned long ofs)
184
+static struct revindex_entry * find_packed_object(struct packed_git *p,
185
+ unsigned int ofs)
186
{
187
int num;
188
int lo, hi;
189
struct pack_revindex *rix;
184
- unsigned long *revindex;
190
+ struct revindex_entry *revindex;
191
num = pack_revindex_ix(p);
192
if (num < 0)
193
die("internal error: pack revindex uninitialized");
@@ -193,10 +199,10 @@ static unsigned long find_packed_object_size(struct packed_git *p,
199
hi = num_packed_objects(p) + 1;
200
do {
201
int mi = (lo + hi) / 2;
196
- if (revindex[mi] == ofs) {
197
- return revindex[mi+1] - ofs;
202
+ if (revindex[mi].offset == ofs) {
203
+ return revindex + mi;
204
}
199
- else if (ofs < revindex[mi])
205
+ else if (ofs < revindex[mi].offset)
206
hi = mi;
207
else
208
lo = mi + 1;
@@ -204,6 +210,20 @@ static unsigned long find_packed_object_size(struct packed_git *p,
210
die("internal error: pack revindex corrupt");
211
}
212
213
+static unsigned long find_packed_object_size(struct packed_git *p,
214
+ unsigned long ofs)
215
+{
216
+ struct revindex_entry *entry = find_packed_object(p, ofs);
217
+ return entry[1].offset - ofs;
218
+}
219
+
220
+static unsigned char *find_packed_object_name(struct packed_git *p,
221
+ unsigned long ofs)
222
+{
223
+ struct revindex_entry *entry = find_packed_object(p, ofs);
224
+ return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
225
+}
226
+
227
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
228
{
229
unsigned long othersize, delta_size;
@@ -249,6 +269,10 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
269
return n;
270
}
271
272
+/*
273
+ * we are going to reuse the existing object data as is. make
274
+ * sure it is not corrupt.
275
+ */
276
static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
277
{
278
z_stream stream;
@@ -280,32 +304,6 @@ static int check_inflate(unsigned char *data, unsigned long len, unsigned long e
304
return st;
305
}
306
283
-/*
284
- * we are going to reuse the existing pack entry data. make
285
- * sure it is not corrupt.
286
- */
287
-static int revalidate_pack_entry(struct object_entry *entry, unsigned char *data, unsigned long len)
288
-{
289
- enum object_type type;
290
- unsigned long size, used;
291
-
292
- if (pack_to_stdout)
293
- return 0;
294
-
295
- /* the caller has already called use_packed_git() for us,
296
- * so it is safe to access the pack data from mmapped location.
297
- * make sure the entry inflates correctly.
298
- */
299
- used = unpack_object_header_gently(data, len, &type, &size);
300
- if (!used)
301
- return -1;
302
- if (type == OBJ_REF_DELTA)
303
- used += 20; /* skip base object name */
304
- data += used;
305
- len -= used;
306
- return check_inflate(data, len, entry->size);
307
-}
308
-
307
static int revalidate_loose_object(struct object_entry *entry,
308
unsigned char *map,
309
unsigned long mapsize)
@@ -339,7 +337,7 @@ static unsigned long write_object(struct sha1file *f,
337
obj_type = entry->type;
338
if (! entry->in_pack)
339
to_reuse = 0; /* can't reuse what we don't have */
342
- else if (obj_type == OBJ_REF_DELTA)
340
+ else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
341
to_reuse = 1; /* check_object() decided it for us */
342
else if (obj_type != entry->in_pack_type)
343
to_reuse = 0; /* pack has delta which is unusable */
@@ -415,18 +413,38 @@ static unsigned long write_object(struct sha1file *f,
413
}
414
else {
415
struct packed_git *p = entry->in_pack;
418
- use_packed_git(p);
416
420
- datalen = find_packed_object_size(p, entry->in_pack_offset);
421
- buf = (char *) p->pack_base + entry->in_pack_offset;
417
+ if (entry->delta) {
418
+ obj_type = (allow_ofs_delta && entry->delta->offset) ?
419
+ OBJ_OFS_DELTA : OBJ_REF_DELTA;
420
+ reused_delta++;
421
+ }
422
+ hdrlen = encode_header(obj_type, entry->size, header);
423
+ sha1write(f, header, hdrlen);
424
+ if (obj_type == OBJ_OFS_DELTA) {
425
+ unsigned long ofs = entry->offset - entry->delta->offset;
426
+ unsigned pos = sizeof(header) - 1;
427
+ header[pos] = ofs & 127;
428
+ while (ofs >>= 7)
429
+ header[--pos] = 128 | (--ofs & 127);
430
+ sha1write(f, header + pos, sizeof(header) - pos);
431
+ hdrlen += sizeof(header) - pos;
432
+ } else if (obj_type == OBJ_REF_DELTA) {
433
+ sha1write(f, entry->delta->sha1, 20);
434
+ hdrlen += 20;
435
+ }
436
423
- if (revalidate_pack_entry(entry, buf, datalen))
437
+ use_packed_git(p);
438
+ buf = (char *) p->pack_base
439
+ + entry->in_pack_offset
440
+ + entry->in_pack_header_size;
441
+ datalen = find_packed_object_size(p, entry->in_pack_offset)
442
+ - entry->in_pack_header_size;
443
+//fprintf(stderr, "reusing %d at %d header %d size %d\n", obj_type, entry->in_pack_offset, entry->in_pack_header_size, datalen);
444
+ if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
445
die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
446
sha1write(f, buf, datalen);
447
unuse_packed_git(p);
427
- hdrlen = 0; /* not really */
428
- if (obj_type == OBJ_REF_DELTA)
429
- reused_delta++;
448
reused++;
449
}
450
if (entry->delta)
@@ -914,26 +932,64 @@ static void check_object(struct object_entry *entry)
932
char type[20];
933
934
if (entry->in_pack && !entry->preferred_base) {
917
- unsigned char base[20];
918
- unsigned long size;
919
- struct object_entry *base_entry;
935
+ struct packed_git *p = entry->in_pack;
936
+ unsigned long left = p->pack_size - entry->in_pack_offset;
937
+ unsigned long size, used;
938
+ unsigned char *buf;
939
+ struct object_entry *base_entry = NULL;
940
+
941
+ use_packed_git(p);
942
+ buf = p->pack_base;
943
+ buf += entry->in_pack_offset;
944
945
/* We want in_pack_type even if we do not reuse delta.
946
* There is no point not reusing non-delta representations.
947
*/
924
- check_reuse_pack_delta(entry->in_pack,
925
- entry->in_pack_offset,
926
- base, &size,
927
- &entry->in_pack_type);
948
+ used = unpack_object_header_gently(buf, left,
949
+ &entry->in_pack_type, &size);
950
+ if (!used || left - used <= 20)
951
+ die("corrupt pack for %s", sha1_to_hex(entry->sha1));
952
953
/* Check if it is delta, and the base is also an object
954
* we are going to pack. If so we will reuse the existing
955
* delta.
956
*/
933
- if (!no_reuse_delta &&
934
- entry->in_pack_type == OBJ_REF_DELTA &&
935
- (base_entry = locate_object_entry(base)) &&
936
- (!base_entry->preferred_base)) {
957
+ if (!no_reuse_delta) {
958
+ unsigned char c, *base_name;
959
+ unsigned long ofs;
960
+ /* there is at least 20 bytes left in the pack */
961
+ switch (entry->in_pack_type) {
962
+ case OBJ_REF_DELTA:
963
+ base_name = buf + used;
964
+ used += 20;
965
+ break;
966
+ case OBJ_OFS_DELTA:
967
+ c = buf[used++];
968
+ ofs = c & 127;
969
+ while (c & 128) {
970
+ ofs += 1;
971
+ if (!ofs || ofs & ~(~0UL >> 7))
972
+ die("delta base offset overflow in pack for %s",
973
+ sha1_to_hex(entry->sha1));
974
+ c = buf[used++];
975
+ ofs = (ofs << 7) + (c & 127);
976
+ }
977
+ if (ofs >= entry->in_pack_offset)
978
+ die("delta base offset out of bound for %s",
979
+ sha1_to_hex(entry->sha1));
980
+ ofs = entry->in_pack_offset - ofs;
981
+ base_name = find_packed_object_name(p, ofs);
982
+ break;
983
+ default:
984
+ base_name = NULL;
985
+ }
986
+ if (base_name)
987
+ base_entry = locate_object_entry(base_name);
988
+ }
989
+ unuse_packed_git(p);
990
+ entry->in_pack_header_size = used;
991
+
992
+ if (base_entry && !base_entry->preferred_base) {
993
994
/* Depth value does not matter - find_deltas()
995
* will never consider reused delta as the
@@ -942,9 +998,9 @@ static void check_object(struct object_entry *entry)
998
*/
999
1000
/* uncompressed size of the delta data */
945
- entry->size = entry->delta_size = size;
1001
+ entry->size = size;
1002
entry->delta = base_entry;
947
- entry->type = OBJ_REF_DELTA;
1003
+ entry->type = entry->in_pack_type;
1004
1005
entry->delta_sibling = base_entry->delta_child;
1006
base_entry->delta_child = entry;
@@ -1500,7 +1556,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1556
continue;
1557
}
1558
if (!strcmp("--delta-base-offset", arg)) {
1503
- allow_ofs_delta = no_reuse_delta = 1;
1559
+ allow_ofs_delta = 1;
1560
continue;
1561
}
1562
if (!strcmp("--stdout", arg)) {
pack.h
-3
@@ -16,7 +16,4 @@ struct pack_header {
16
};
17
18
extern int verify_pack(struct packed_git *, int);
19
-extern int check_reuse_pack_delta(struct packed_git *, unsigned long,
20
- unsigned char *, unsigned long *,
21
- enum object_type *);
19
#endif
sha1_file.c
-19
@@ -1000,25 +1000,6 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of
1000
return offset + used;
1001
}
1002
1003
-int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
1004
- unsigned char *base, unsigned long *sizep,
1005
- enum object_type *kindp)
1006
-{
1007
- unsigned long ptr;
1008
- int status = -1;
1009
-
1010
- use_packed_git(p);
1011
- ptr = offset;
1012
- ptr = unpack_object_header(p, ptr, kindp, sizep);
1013
- if (*kindp != OBJ_REF_DELTA)
1014
- goto done;
1015
- hashcpy(base, (unsigned char *) p->pack_base + ptr);
1016
- status = 0;
1017
- done:
1018
- unuse_packed_git(p);
1019
- return status;
1020
-}
1021
-
1003
void packed_object_info_detail(struct packed_git *p,
1004
unsigned long offset,
1005
char *type,