more lightweight revalidation while reusing deflated stream in packing
When copying from an existing pack and when copying from a loose object with new style header, the code makes sure that the piece we are going to copy out inflates well and inflate() consumes the data in full while doing so. The check to see if the xdelta really apply is quite expensive as you described, because you would need to have the image of the base object which can be represented as a delta against something else. Signed-off-by: Junio C Hamano <junkio@cox.net>
Junio C Hamano committed
Sep 3, 2006 at 21:09 UTC
72518e9c2623af0b5de864a7b66208ea94aacadb
4 files changed
+65
-41
builtin-pack-objects.c
+52
-29
@@ -243,41 +243,61 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
243
return n;
244
}
245
246
-static int revalidate_one(struct object_entry *entry,
247
- void *data, char *type, unsigned long size)
246
+static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
247
{
249
- int err;
250
- if ((!data) ||
251
- ((entry->type != OBJ_DELTA) &&
252
- ( (size != entry->size) ||
253
- strcmp(type_names[entry->type], type))))
254
- err = -1;
255
- else
256
- err = check_sha1_signature(entry->sha1, data, size, type);
257
- free(data);
258
- return err;
248
+ z_stream stream;
249
+ unsigned char fakebuf[4096];
250
+ int st;
251
+
252
+ memset(&stream, 0, sizeof(stream));
253
+ stream.next_in = data;
254
+ stream.avail_in = len;
255
+ stream.next_out = fakebuf;
256
+ stream.avail_out = sizeof(fakebuf);
257
+ inflateInit(&stream);
258
+
259
+ while (1) {
260
+ st = inflate(&stream, Z_FINISH);
261
+ if (st == Z_STREAM_END || st == Z_OK) {
262
+ st = (stream.total_out == expect &&
263
+ stream.total_in == len) ? 0 : -1;
264
+ break;
265
+ }
266
+ if (st != Z_BUF_ERROR) {
267
+ st = -1;
268
+ break;
269
+ }
270
+ stream.next_out = fakebuf;
271
+ stream.avail_out = sizeof(fakebuf);
272
+ }
273
+ inflateEnd(&stream);
274
+ return st;
275
}
276
277
/*
278
* we are going to reuse the existing pack entry data. make
279
* sure it is not corrupt.
280
*/
265
-static int revalidate_pack_entry(struct object_entry *entry)
281
+static int revalidate_pack_entry(struct object_entry *entry, unsigned char *data, unsigned long len)
282
{
267
- void *data;
268
- char type[20];
269
- unsigned long size;
270
- struct pack_entry e;
283
+ enum object_type type;
284
+ unsigned long size, used;
285
286
if (pack_to_stdout)
287
return 0;
288
275
- e.p = entry->in_pack;
276
- e.offset = entry->in_pack_offset;
277
-
278
- /* the caller has already called use_packed_git() for us */
279
- data = unpack_entry_gently(&e, type, &size);
280
- return revalidate_one(entry, data, type, size);
289
+ /* the caller has already called use_packed_git() for us,
290
+ * so it is safe to access the pack data from mmapped location.
291
+ * make sure the entry inflates correctly.
292
+ */
293
+ used = unpack_object_header_gently(data, len, &type, &size);
294
+ if (!used)
295
+ return -1;
296
+ if (type == OBJ_DELTA)
297
+ used += 20; /* skip base object name */
298
+ data += used;
299
+ len -= used;
300
+ return check_inflate(data, len, entry->size);
301
}
302
303
static int revalidate_loose_object(struct object_entry *entry,
@@ -285,15 +305,18 @@ static int revalidate_loose_object(struct object_entry *entry,
305
unsigned long mapsize)
306
{
307
/* we already know this is a loose object with new type header. */
288
- void *data;
289
- char type[20];
290
- unsigned long size;
308
+ enum object_type type;
309
+ unsigned long size, used;
310
311
if (pack_to_stdout)
312
return 0;
313
295
- data = unpack_sha1_file(map, mapsize, type, &size);
296
- return revalidate_one(entry, data, type, size);
314
+ used = unpack_object_header_gently(map, mapsize, &type, &size);
315
+ if (!used)
316
+ return -1;
317
+ map += used;
318
+ mapsize -= used;
319
+ return check_inflate(map, mapsize, size);
320
}
321
322
static unsigned long write_object(struct sha1file *f,
@@ -377,7 +400,7 @@ static unsigned long write_object(struct sha1file *f,
400
datalen = find_packed_object_size(p, entry->in_pack_offset);
401
buf = (char *) p->pack_base + entry->in_pack_offset;
402
380
- if (revalidate_pack_entry(entry))
403
+ if (revalidate_pack_entry(entry, buf, datalen))
404
die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
405
sha1write(f, buf, datalen);
406
unuse_packed_git(p);
cache.h
+12
@@ -267,6 +267,17 @@ extern int legacy_loose_object(unsigned char *);
267
extern int has_pack_file(const unsigned char *sha1);
268
extern int has_pack_index(const unsigned char *sha1);
269
270
+enum object_type {
271
+ OBJ_NONE = 0,
272
+ OBJ_COMMIT = 1,
273
+ OBJ_TREE = 2,
274
+ OBJ_BLOB = 3,
275
+ OBJ_TAG = 4,
276
+ /* 5/6 for future expansion */
277
+ OBJ_DELTA = 7,
278
+ OBJ_BAD,
279
+};
280
+
281
/* Convert to/from hex/sha1 representation */
282
#define MINIMUM_ABBREV 4
283
#define DEFAULT_ABBREV 7
@@ -374,6 +385,7 @@ extern int num_packed_objects(const struct packed_git *p);
385
extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*);
386
extern int find_pack_entry_one(const unsigned char *, struct pack_entry *, struct packed_git *);
387
extern void *unpack_entry_gently(struct pack_entry *, char *, unsigned long *);
388
+extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
389
extern void packed_object_info_detail(struct pack_entry *, char *, unsigned long *, unsigned long *, unsigned int *, unsigned char *);
390
391
/* Dumb servers support */
object.h
-11
@@ -27,17 +27,6 @@ struct object_array {
27
/*
28
* The object type is stored in 3 bits.
29
*/
30
-enum object_type {
31
- OBJ_NONE = 0,
32
- OBJ_COMMIT = 1,
33
- OBJ_TREE = 2,
34
- OBJ_BLOB = 3,
35
- OBJ_TAG = 4,
36
- /* 5/6 for future expansion */
37
- OBJ_DELTA = 7,
38
- OBJ_BAD,
39
-};
40
-
30
struct object {
31
unsigned parsed : 1;
32
unsigned used : 1;
sha1_file.c
+1
-1
@@ -711,7 +711,7 @@ int legacy_loose_object(unsigned char *map)
711
return 0;
712
}
713
714
-static unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
714
+unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
715
{
716
unsigned shift;
717
unsigned char c;