@matheusbsbs / matheus-bagliano / commits / 780e6e735b
Fork 0

make pack data reuse compatible with both delta types

This is the missing part to git-pack-objects allowing it to reuse delta data to/from any of the two delta types. It can reuse delta from any type, and it outputs base offsets when --allow-delta-base-offset is provided and the base is also included in the pack. Otherwise it outputs base sha1 references just like it always did. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Sep 22, 2006 at 21:25 UTC 780e6e735be189097dad4b223d8edeb18cce1928
3 files changed +130 -96
builtin-pack-objects.c
+130 -74
@@ -29,6 +29,7 @@ struct object_entry {
29 enum object_type type;
30 enum object_type in_pack_type; /* could be delta */
31 unsigned long delta_size; /* delta data size (uncompressed) */
32 +#define in_pack_header_size delta_size /* only when reusing pack data */
33 struct object_entry *delta; /* delta base object */
34 struct packed_git *in_pack; /* already in pack */
35 unsigned int in_pack_offset;
@@ -86,17 +87,25 @@ static int object_ix_hashsz;
87 * Pack index for existing packs give us easy access to the offsets into
88 * corresponding pack file where each object's data starts, but the entries
89 * do not store the size of the compressed representation (uncompressed
89 - * size is easily available by examining the pack entry header). We build
90 - * a hashtable of existing packs (pack_revindex), and keep reverse index
91 - * here -- pack index file is sorted by object name mapping to offset; this
92 - * pack_revindex[].revindex array is an ordered list of offsets, so if you
93 - * know the offset of an object, next offset is where its packed
94 - * representation ends.
90 + * size is easily available by examining the pack entry header). It is
91 + * also rather expensive to find the sha1 for an object given its offset.
92 + *
93 + * We build a hashtable of existing packs (pack_revindex), and keep reverse
94 + * index here -- pack index file is sorted by object name mapping to offset;
95 + * this pack_revindex[].revindex array is a list of offset/index_nr pairs
96 + * ordered by offset, so if you know the offset of an object, next offset
97 + * is where its packed representation ends and the index_nr can be used to
98 + * get the object sha1 from the main index.
99 */
100 +struct revindex_entry {
101 + unsigned int offset;
102 + unsigned int nr;
103 +};
104 struct pack_revindex {
105 struct packed_git *p;
98 - unsigned long *revindex;
99 -} *pack_revindex = NULL;
106 + struct revindex_entry *revindex;
107 +};
108 +static struct pack_revindex *pack_revindex;
109 static int pack_revindex_hashsz;
110
111 /*
@@ -143,14 +152,9 @@ static void prepare_pack_ix(void)
152
153 static int cmp_offset(const void *a_, const void *b_)
154 {
146 - unsigned long a = *(unsigned long *) a_;
147 - unsigned long b = *(unsigned long *) b_;
148 - if (a < b)
149 - return -1;
150 - else if (a == b)
151 - return 0;
152 - else
153 - return 1;
155 + const struct revindex_entry *a = a_;
156 + const struct revindex_entry *b = b_;
157 + return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
158 }
159
160 /*
@@ -163,25 +167,27 @@ static void prepare_pack_revindex(struct pack_revindex *rix)
167 int i;
168 void *index = p->index_base + 256;
169
166 - rix->revindex = xmalloc(sizeof(unsigned long) * (num_ent + 1));
170 + rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
171 for (i = 0; i < num_ent; i++) {
172 unsigned int hl = *((unsigned int *)((char *) index + 24*i));
169 - rix->revindex[i] = ntohl(hl);
173 + rix->revindex[i].offset = ntohl(hl);
174 + rix->revindex[i].nr = i;
175 }
176 /* This knows the pack format -- the 20-byte trailer
177 * follows immediately after the last object data.
178 */
174 - rix->revindex[num_ent] = p->pack_size - 20;
175 - qsort(rix->revindex, num_ent, sizeof(unsigned long), cmp_offset);
179 + rix->revindex[num_ent].offset = p->pack_size - 20;
180 + rix->revindex[num_ent].nr = -1;
181 + qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
182 }
183
178 -static unsigned long find_packed_object_size(struct packed_git *p,
179 - unsigned long ofs)
184 +static struct revindex_entry * find_packed_object(struct packed_git *p,
185 + unsigned int ofs)
186 {
187 int num;
188 int lo, hi;
189 struct pack_revindex *rix;
184 - unsigned long *revindex;
190 + struct revindex_entry *revindex;
191 num = pack_revindex_ix(p);
192 if (num < 0)
193 die("internal error: pack revindex uninitialized");
@@ -193,10 +199,10 @@ static unsigned long find_packed_object_size(struct packed_git *p,
199 hi = num_packed_objects(p) + 1;
200 do {
201 int mi = (lo + hi) / 2;
196 - if (revindex[mi] == ofs) {
197 - return revindex[mi+1] - ofs;
202 + if (revindex[mi].offset == ofs) {
203 + return revindex + mi;
204 }
199 - else if (ofs < revindex[mi])
205 + else if (ofs < revindex[mi].offset)
206 hi = mi;
207 else
208 lo = mi + 1;
@@ -204,6 +210,20 @@ static unsigned long find_packed_object_size(struct packed_git *p,
210 die("internal error: pack revindex corrupt");
211 }
212
213 +static unsigned long find_packed_object_size(struct packed_git *p,
214 + unsigned long ofs)
215 +{
216 + struct revindex_entry *entry = find_packed_object(p, ofs);
217 + return entry[1].offset - ofs;
218 +}
219 +
220 +static unsigned char *find_packed_object_name(struct packed_git *p,
221 + unsigned long ofs)
222 +{
223 + struct revindex_entry *entry = find_packed_object(p, ofs);
224 + return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
225 +}
226 +
227 static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
228 {
229 unsigned long othersize, delta_size;
@@ -249,6 +269,10 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
269 return n;
270 }
271
272 +/*
273 + * we are going to reuse the existing object data as is. make
274 + * sure it is not corrupt.
275 + */
276 static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
277 {
278 z_stream stream;
@@ -280,32 +304,6 @@ static int check_inflate(unsigned char *data, unsigned long len, unsigned long e
304 return st;
305 }
306
283 -/*
284 - * we are going to reuse the existing pack entry data. make
285 - * sure it is not corrupt.
286 - */
287 -static int revalidate_pack_entry(struct object_entry *entry, unsigned char *data, unsigned long len)
288 -{
289 - enum object_type type;
290 - unsigned long size, used;
291 -
292 - if (pack_to_stdout)
293 - return 0;
294 -
295 - /* the caller has already called use_packed_git() for us,
296 - * so it is safe to access the pack data from mmapped location.
297 - * make sure the entry inflates correctly.
298 - */
299 - used = unpack_object_header_gently(data, len, &type, &size);
300 - if (!used)
301 - return -1;
302 - if (type == OBJ_REF_DELTA)
303 - used += 20; /* skip base object name */
304 - data += used;
305 - len -= used;
306 - return check_inflate(data, len, entry->size);
307 -}
308 -
307 static int revalidate_loose_object(struct object_entry *entry,
308 unsigned char *map,
309 unsigned long mapsize)
@@ -339,7 +337,7 @@ static unsigned long write_object(struct sha1file *f,
337 obj_type = entry->type;
338 if (! entry->in_pack)
339 to_reuse = 0; /* can't reuse what we don't have */
342 - else if (obj_type == OBJ_REF_DELTA)
340 + else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
341 to_reuse = 1; /* check_object() decided it for us */
342 else if (obj_type != entry->in_pack_type)
343 to_reuse = 0; /* pack has delta which is unusable */
@@ -415,18 +413,38 @@ static unsigned long write_object(struct sha1file *f,
413 }
414 else {
415 struct packed_git *p = entry->in_pack;
418 - use_packed_git(p);
416
420 - datalen = find_packed_object_size(p, entry->in_pack_offset);
421 - buf = (char *) p->pack_base + entry->in_pack_offset;
417 + if (entry->delta) {
418 + obj_type = (allow_ofs_delta && entry->delta->offset) ?
419 + OBJ_OFS_DELTA : OBJ_REF_DELTA;
420 + reused_delta++;
421 + }
422 + hdrlen = encode_header(obj_type, entry->size, header);
423 + sha1write(f, header, hdrlen);
424 + if (obj_type == OBJ_OFS_DELTA) {
425 + unsigned long ofs = entry->offset - entry->delta->offset;
426 + unsigned pos = sizeof(header) - 1;
427 + header[pos] = ofs & 127;
428 + while (ofs >>= 7)
429 + header[--pos] = 128 | (--ofs & 127);
430 + sha1write(f, header + pos, sizeof(header) - pos);
431 + hdrlen += sizeof(header) - pos;
432 + } else if (obj_type == OBJ_REF_DELTA) {
433 + sha1write(f, entry->delta->sha1, 20);
434 + hdrlen += 20;
435 + }
436
423 - if (revalidate_pack_entry(entry, buf, datalen))
437 + use_packed_git(p);
438 + buf = (char *) p->pack_base
439 + + entry->in_pack_offset
440 + + entry->in_pack_header_size;
441 + datalen = find_packed_object_size(p, entry->in_pack_offset)
442 + - entry->in_pack_header_size;
443 +//fprintf(stderr, "reusing %d at %d header %d size %d\n", obj_type, entry->in_pack_offset, entry->in_pack_header_size, datalen);
444 + if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
445 die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
446 sha1write(f, buf, datalen);
447 unuse_packed_git(p);
427 - hdrlen = 0; /* not really */
428 - if (obj_type == OBJ_REF_DELTA)
429 - reused_delta++;
448 reused++;
449 }
450 if (entry->delta)
@@ -914,26 +932,64 @@ static void check_object(struct object_entry *entry)
932 char type[20];
933
934 if (entry->in_pack && !entry->preferred_base) {
917 - unsigned char base[20];
918 - unsigned long size;
919 - struct object_entry *base_entry;
935 + struct packed_git *p = entry->in_pack;
936 + unsigned long left = p->pack_size - entry->in_pack_offset;
937 + unsigned long size, used;
938 + unsigned char *buf;
939 + struct object_entry *base_entry = NULL;
940 +
941 + use_packed_git(p);
942 + buf = p->pack_base;
943 + buf += entry->in_pack_offset;
944
945 /* We want in_pack_type even if we do not reuse delta.
946 * There is no point not reusing non-delta representations.
947 */
924 - check_reuse_pack_delta(entry->in_pack,
925 - entry->in_pack_offset,
926 - base, &size,
927 - &entry->in_pack_type);
948 + used = unpack_object_header_gently(buf, left,
949 + &entry->in_pack_type, &size);
950 + if (!used || left - used <= 20)
951 + die("corrupt pack for %s", sha1_to_hex(entry->sha1));
952
953 /* Check if it is delta, and the base is also an object
954 * we are going to pack. If so we will reuse the existing
955 * delta.
956 */
933 - if (!no_reuse_delta &&
934 - entry->in_pack_type == OBJ_REF_DELTA &&
935 - (base_entry = locate_object_entry(base)) &&
936 - (!base_entry->preferred_base)) {
957 + if (!no_reuse_delta) {
958 + unsigned char c, *base_name;
959 + unsigned long ofs;
960 + /* there is at least 20 bytes left in the pack */
961 + switch (entry->in_pack_type) {
962 + case OBJ_REF_DELTA:
963 + base_name = buf + used;
964 + used += 20;
965 + break;
966 + case OBJ_OFS_DELTA:
967 + c = buf[used++];
968 + ofs = c & 127;
969 + while (c & 128) {
970 + ofs += 1;
971 + if (!ofs || ofs & ~(~0UL >> 7))
972 + die("delta base offset overflow in pack for %s",
973 + sha1_to_hex(entry->sha1));
974 + c = buf[used++];
975 + ofs = (ofs << 7) + (c & 127);
976 + }
977 + if (ofs >= entry->in_pack_offset)
978 + die("delta base offset out of bound for %s",
979 + sha1_to_hex(entry->sha1));
980 + ofs = entry->in_pack_offset - ofs;
981 + base_name = find_packed_object_name(p, ofs);
982 + break;
983 + default:
984 + base_name = NULL;
985 + }
986 + if (base_name)
987 + base_entry = locate_object_entry(base_name);
988 + }
989 + unuse_packed_git(p);
990 + entry->in_pack_header_size = used;
991 +
992 + if (base_entry && !base_entry->preferred_base) {
993
994 /* Depth value does not matter - find_deltas()
995 * will never consider reused delta as the
@@ -942,9 +998,9 @@ static void check_object(struct object_entry *entry)
998 */
999
1000 /* uncompressed size of the delta data */
945 - entry->size = entry->delta_size = size;
1001 + entry->size = size;
1002 entry->delta = base_entry;
947 - entry->type = OBJ_REF_DELTA;
1003 + entry->type = entry->in_pack_type;
1004
1005 entry->delta_sibling = base_entry->delta_child;
1006 base_entry->delta_child = entry;
@@ -1500,7 +1556,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1556 continue;
1557 }
1558 if (!strcmp("--delta-base-offset", arg)) {
1503 - allow_ofs_delta = no_reuse_delta = 1;
1559 + allow_ofs_delta = 1;
1560 continue;
1561 }
1562 if (!strcmp("--stdout", arg)) {
pack.h
-3
@@ -16,7 +16,4 @@ struct pack_header {
16 };
17
18 extern int verify_pack(struct packed_git *, int);
19 -extern int check_reuse_pack_delta(struct packed_git *, unsigned long,
20 - unsigned char *, unsigned long *,
21 - enum object_type *);
19 #endif
sha1_file.c
-19
@@ -1000,25 +1000,6 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of
1000 return offset + used;
1001 }
1002
1003 -int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
1004 - unsigned char *base, unsigned long *sizep,
1005 - enum object_type *kindp)
1006 -{
1007 - unsigned long ptr;
1008 - int status = -1;
1009 -
1010 - use_packed_git(p);
1011 - ptr = offset;
1012 - ptr = unpack_object_header(p, ptr, kindp, sizep);
1013 - if (*kindp != OBJ_REF_DELTA)
1014 - goto done;
1015 - hashcpy(base, (unsigned char *) p->pack_base + ptr);
1016 - status = 0;
1017 - done:
1018 - unuse_packed_git(p);
1019 - return status;
1020 -}
1021 -
1003 void packed_object_info_detail(struct packed_git *p,
1004 unsigned long offset,
1005 char *type,