@matheusbsbs / matheus-bagliano / commits / 079afb18fe
Fork 0

Loop over pack_windows when inflating/accessing data.

When multiple mmaps start getting used for all pack file access it is not possible to get all data associated with a specific object in one contiguous memory region. This limitation prevents simply passing a single address and length to SHA1_Update or to inflate. Instead we need to loop until we have processed all data of interest. As we loop over the data we are always interested in reusing the same window 'cursor', as the prior window will no longer be of any use to us. This allows the use_pack() call to automatically decrement the use count of the prior window before setting up access for us to the next window. Within each loop we need to make use of the available length output parameter of use_pack() to tell us how many bytes are available in the current memory region, as we cannot tell otherwise. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Dec 23, 2006 at 02:34 UTC 079afb18fed078af01bd9ab02e2ebbe5d31893b1
3 files changed +87 -39
builtin-pack-objects.c
+52 -6
@@ -276,7 +276,52 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
276 * we are going to reuse the existing object data as is. make
277 * sure it is not corrupt.
278 */
279 -static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
279 +static int check_pack_inflate(struct packed_git *p,
280 + struct pack_window **w_curs,
281 + unsigned long offset,
282 + unsigned long len,
283 + unsigned long expect)
284 +{
285 + z_stream stream;
286 + unsigned char fakebuf[4096], *in;
287 + int st;
288 +
289 + memset(&stream, 0, sizeof(stream));
290 + inflateInit(&stream);
291 + do {
292 + in = use_pack(p, w_curs, offset, &stream.avail_in);
293 + stream.next_in = in;
294 + stream.next_out = fakebuf;
295 + stream.avail_out = sizeof(fakebuf);
296 + st = inflate(&stream, Z_FINISH);
297 + offset += stream.next_in - in;
298 + } while (st == Z_OK || st == Z_BUF_ERROR);
299 + inflateEnd(&stream);
300 + return (st == Z_STREAM_END &&
301 + stream.total_out == expect &&
302 + stream.total_in == len) ? 0 : -1;
303 +}
304 +
305 +static void copy_pack_data(struct sha1file *f,
306 + struct packed_git *p,
307 + struct pack_window **w_curs,
308 + unsigned long offset,
309 + unsigned long len)
310 +{
311 + unsigned char *in;
312 + unsigned int avail;
313 +
314 + while (len) {
315 + in = use_pack(p, w_curs, offset, &avail);
316 + if (avail > len)
317 + avail = len;
318 + sha1write(f, in, avail);
319 + offset += avail;
320 + len -= avail;
321 + }
322 +}
323 +
324 +static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)
325 {
326 z_stream stream;
327 unsigned char fakebuf[4096];
@@ -323,7 +368,7 @@ static int revalidate_loose_object(struct object_entry *entry,
368 return -1;
369 map += used;
370 mapsize -= used;
326 - return check_inflate(map, mapsize, size);
371 + return check_loose_inflate(map, mapsize, size);
372 }
373
374 static unsigned long write_object(struct sha1file *f,
@@ -417,6 +462,7 @@ static unsigned long write_object(struct sha1file *f,
462 else {
463 struct packed_git *p = entry->in_pack;
464 struct pack_window *w_curs = NULL;
465 + unsigned long offset;
466
467 if (entry->delta) {
468 obj_type = (allow_ofs_delta && entry->delta->offset) ?
@@ -438,13 +484,13 @@ static unsigned long write_object(struct sha1file *f,
484 hdrlen += 20;
485 }
486
441 - buf = use_pack(p, &w_curs, entry->in_pack_offset
442 - + entry->in_pack_header_size, NULL);
487 + offset = entry->in_pack_offset + entry->in_pack_header_size;
488 datalen = find_packed_object_size(p, entry->in_pack_offset)
489 - entry->in_pack_header_size;
445 - if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
490 + if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
491 + offset, datalen, entry->size))
492 die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
447 - sha1write(f, buf, datalen);
493 + copy_pack_data(f, p, &w_curs, offset, datalen);
494 unuse_pack(&w_curs);
495 reused++;
496 }
pack-check.c
+20 -26
@@ -8,39 +8,38 @@ static int verify_packfile(struct packed_git *p,
8 void *index_base = p->index_base;
9 SHA_CTX ctx;
10 unsigned char sha1[20];
11 - unsigned long pack_size = p->pack_size;
12 - void *pack_base;
13 - struct pack_header *hdr;
11 + unsigned long offset = 0, pack_sig = p->pack_size - 20;
12 int nr_objects, err, i;
13
16 - /* Header consistency check */
17 - pack_base = use_pack(p, w_curs, 0, NULL);
18 - hdr = (struct pack_header*)pack_base;
19 - if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
20 - return error("Packfile %s signature mismatch", p->pack_name);
21 - if (!pack_version_ok(hdr->hdr_version))
22 - return error("Packfile version %d unsupported",
23 - ntohl(hdr->hdr_version));
24 - nr_objects = ntohl(hdr->hdr_entries);
25 - if (num_packed_objects(p) != nr_objects)
26 - return error("Packfile claims to have %d objects, "
27 - "while idx size expects %d", nr_objects,
28 - num_packed_objects(p));
14 + /* Note that the pack header checks are actually performed by
15 + * use_pack when it first opens the pack file. If anything
16 + * goes wrong during those checks then the call will die out
17 + * immediately.
18 + */
19
20 SHA1_Init(&ctx);
31 - SHA1_Update(&ctx, pack_base, pack_size - 20);
21 + while (offset < pack_sig) {
22 + unsigned int remaining;
23 + unsigned char *in = use_pack(p, w_curs, offset, &remaining);
24 + offset += remaining;
25 + if (offset > pack_sig)
26 + remaining -= offset - pack_sig;
27 + SHA1_Update(&ctx, in, remaining);
28 + }
29 SHA1_Final(sha1, &ctx);
33 - if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20))
30 + if (hashcmp(sha1, use_pack(p, w_curs, pack_sig, NULL)))
31 return error("Packfile %s SHA1 mismatch with itself",
32 p->pack_name);
33 if (hashcmp(sha1, (unsigned char *)index_base + index_size - 40))
34 return error("Packfile %s SHA1 mismatch with idx",
35 p->pack_name);
36 + unuse_pack(w_curs);
37
38 /* Make sure everything reachable from idx is valid. Since we
39 * have verified that nr_objects matches between idx and pack,
40 * we do not do scan-streaming check on the pack file.
41 */
42 + nr_objects = num_packed_objects(p);
43 for (i = err = 0; i < nr_objects; i++) {
44 unsigned char sha1[20];
45 void *data;
@@ -73,15 +72,12 @@ static int verify_packfile(struct packed_git *p,
72
73 #define MAX_CHAIN 40
74
76 -static void show_pack_info(struct packed_git *p,
77 - struct pack_window **w_curs)
75 +static void show_pack_info(struct packed_git *p)
76 {
79 - struct pack_header *hdr;
77 int nr_objects, i;
78 unsigned int chain_histogram[MAX_CHAIN];
79
83 - hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL);
84 - nr_objects = ntohl(hdr->hdr_entries);
80 + nr_objects = num_packed_objects(p);
81 memset(chain_histogram, 0, sizeof(chain_histogram));
82
83 for (i = 0; i < nr_objects; i++) {
@@ -153,9 +149,7 @@ int verify_pack(struct packed_git *p, int verbose)
149 if (ret)
150 printf("%s: bad\n", p->pack_name);
151 else {
156 - struct pack_window *w_curs = NULL;
157 - show_pack_info(p, &w_curs);
158 - unuse_pack(&w_curs);
152 + show_pack_info(p);
153 printf("%s: ok\n", p->pack_name);
154 }
155 }
sha1_file.c
+15 -7
@@ -962,19 +962,23 @@ static int packed_delta_info(struct packed_git *p,
962
963 if (sizep) {
964 const unsigned char *data;
965 - unsigned char delta_head[20];
965 + unsigned char delta_head[20], *in;
966 unsigned long result_size;
967 z_stream stream;
968 int st;
969
970 memset(&stream, 0, sizeof(stream));
971 -
972 - stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);
971 stream.next_out = delta_head;
972 stream.avail_out = sizeof(delta_head);
973
974 inflateInit(&stream);
977 - st = inflate(&stream, Z_FINISH);
975 + do {
976 + in = use_pack(p, w_curs, offset, &stream.avail_in);
977 + stream.next_in = in;
978 + st = inflate(&stream, Z_FINISH);
979 + offset += stream.next_in - in;
980 + } while ((st == Z_OK || st == Z_BUF_ERROR)
981 + && stream.total_out < sizeof(delta_head));
982 inflateEnd(&stream);
983 if ((st != Z_STREAM_END) &&
984 stream.total_out != sizeof(delta_head))
@@ -1103,17 +1107,21 @@ static void *unpack_compressed_entry(struct packed_git *p,
1107 {
1108 int st;
1109 z_stream stream;
1106 - unsigned char *buffer;
1110 + unsigned char *buffer, *in;
1111
1112 buffer = xmalloc(size + 1);
1113 buffer[size] = 0;
1114 memset(&stream, 0, sizeof(stream));
1111 - stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);
1115 stream.next_out = buffer;
1116 stream.avail_out = size;
1117
1118 inflateInit(&stream);
1116 - st = inflate(&stream, Z_FINISH);
1119 + do {
1120 + in = use_pack(p, w_curs, offset, &stream.avail_in);
1121 + stream.next_in = in;
1122 + st = inflate(&stream, Z_FINISH);
1123 + offset += stream.next_in - in;
1124 + } while (st == Z_OK || st == Z_BUF_ERROR);
1125 inflateEnd(&stream);
1126 if ((st != Z_STREAM_END) || stream.total_out != size) {
1127 free(buffer);