@matheusbsbs / matheus-bagliano / commits / 2d477051ef
Fork 0

add the capability for index-pack to read from a stream

This patch only adds the streaming capability to index-pack. Although the code is different it has the exact same functionality as before to make sure nothing broke. This is in preparation for receiving packs over the net, parse them on the fly, fix them up if they are "thin" packs, and keep the resulting pack instead of exploding it into loose objects. But such functionality should come separately. One immediate advantage of this patch is that index-pack can now deal with packs up to 4GB in size even on 32-bit architectures since the pack is not entirely mmap()'d all at once anymore. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Oct 20, 2006 at 14:45 UTC 2d477051ef260aad352d63fc7d9c07e4ebb4359b
1 file changed +143 -101
index-pack.c
+143 -101
@@ -13,6 +13,8 @@ static const char index_pack_usage[] =
13 struct object_entry
14 {
15 unsigned long offset;
16 + unsigned long size;
17 + unsigned int hdr_size;
18 enum object_type type;
19 enum object_type real_type;
20 unsigned char sha1[20];
@@ -36,51 +38,68 @@ struct delta_entry
38 };
39
40 static const char *pack_name;
39 -static unsigned char *pack_base;
40 -static unsigned long pack_size;
41 static struct object_entry *objects;
42 static struct delta_entry *deltas;
43 static int nr_objects;
44 static int nr_deltas;
45
46 -static void open_pack_file(void)
46 +/* We always read in 4kB chunks. */
47 +static unsigned char input_buffer[4096];
48 +static unsigned long input_offset, input_len, consumed_bytes;
49 +static SHA_CTX input_ctx;
50 +static int input_fd;
51 +
52 +/*
53 + * Make sure at least "min" bytes are available in the buffer, and
54 + * return the pointer to the buffer.
55 + */
56 +static void * fill(int min)
57 {
48 - int fd;
49 - struct stat st;
58 + if (min <= input_len)
59 + return input_buffer + input_offset;
60 + if (min > sizeof(input_buffer))
61 + die("cannot fill %d bytes", min);
62 + if (input_offset) {
63 + SHA1_Update(&input_ctx, input_buffer, input_offset);
64 + memcpy(input_buffer, input_buffer + input_offset, input_len);
65 + input_offset = 0;
66 + }
67 + do {
68 + int ret = xread(input_fd, input_buffer + input_len,
69 + sizeof(input_buffer) - input_len);
70 + if (ret <= 0) {
71 + if (!ret)
72 + die("early EOF");
73 + die("read error on input: %s", strerror(errno));
74 + }
75 + input_len += ret;
76 + } while (input_len < min);
77 + return input_buffer;
78 +}
79 +
80 +static void use(int bytes)
81 +{
82 + if (bytes > input_len)
83 + die("used more bytes than were available");
84 + input_len -= bytes;
85 + input_offset += bytes;
86 + consumed_bytes += bytes;
87 +}
88
51 - fd = open(pack_name, O_RDONLY);
52 - if (fd < 0)
89 +static void open_pack_file(void)
90 +{
91 + input_fd = open(pack_name, O_RDONLY);
92 + if (input_fd < 0)
93 die("cannot open packfile '%s': %s", pack_name,
94 strerror(errno));
55 - if (fstat(fd, &st)) {
56 - int err = errno;
57 - close(fd);
58 - die("cannot fstat packfile '%s': %s", pack_name,
59 - strerror(err));
60 - }
61 - pack_size = st.st_size;
62 - pack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
63 - if (pack_base == MAP_FAILED) {
64 - int err = errno;
65 - close(fd);
66 - die("cannot mmap packfile '%s': %s", pack_name,
67 - strerror(err));
68 - }
69 - close(fd);
95 + SHA1_Init(&input_ctx);
96 }
97
98 static void parse_pack_header(void)
99 {
74 - const struct pack_header *hdr;
75 - unsigned char sha1[20];
76 - SHA_CTX ctx;
77 -
78 - /* Ensure there are enough bytes for the header and final SHA1 */
79 - if (pack_size < sizeof(struct pack_header) + 20)
80 - die("packfile '%s' is too small", pack_name);
100 + struct pack_header *hdr = fill(sizeof(struct pack_header));
101
102 /* Header consistency check */
83 - hdr = (void *)pack_base;
103 if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
104 die("packfile '%s' signature mismatch", pack_name);
105 if (!pack_version_ok(hdr->hdr_version))
@@ -88,13 +107,8 @@ static void parse_pack_header(void)
107 pack_name, ntohl(hdr->hdr_version));
108
109 nr_objects = ntohl(hdr->hdr_entries);
91 -
92 - /* Check packfile integrity */
93 - SHA1_Init(&ctx);
94 - SHA1_Update(&ctx, pack_base, pack_size - 20);
95 - SHA1_Final(sha1, &ctx);
96 - if (hashcmp(sha1, pack_base + pack_size - 20))
97 - die("packfile '%s' SHA1 mismatch", pack_name);
110 + use(sizeof(struct pack_header));
111 + /*fprintf(stderr, "Indexing %d objects\n", nr_objects);*/
112 }
113
114 static void bad_object(unsigned long offset, const char *format,
@@ -112,85 +126,78 @@ static void bad_object(unsigned long offset, const char *format, ...)
126 pack_name, offset, buf);
127 }
128
115 -static void *unpack_entry_data(unsigned long offset,
116 - unsigned long *current_pos, unsigned long size)
129 +static void *unpack_entry_data(unsigned long offset, unsigned long size)
130 {
118 - unsigned long pack_limit = pack_size - 20;
119 - unsigned long pos = *current_pos;
131 z_stream stream;
132 void *buf = xmalloc(size);
133
134 memset(&stream, 0, sizeof(stream));
135 stream.next_out = buf;
136 stream.avail_out = size;
126 - stream.next_in = pack_base + pos;
127 - stream.avail_in = pack_limit - pos;
137 + stream.next_in = fill(1);
138 + stream.avail_in = input_len;
139 inflateInit(&stream);
140
141 for (;;) {
142 int ret = inflate(&stream, 0);
132 - if (ret == Z_STREAM_END)
143 + use(input_len - stream.avail_in);
144 + if (stream.total_out == size && ret == Z_STREAM_END)
145 break;
146 if (ret != Z_OK)
147 bad_object(offset, "inflate returned %d", ret);
148 + stream.next_in = fill(1);
149 + stream.avail_in = input_len;
150 }
151 inflateEnd(&stream);
138 - if (stream.total_out != size)
139 - bad_object(offset, "size mismatch (expected %lu, got %lu)",
140 - size, stream.total_out);
141 - *current_pos = pack_limit - stream.avail_in;
152 return buf;
153 }
154
145 -static void *unpack_raw_entry(unsigned long offset,
146 - enum object_type *obj_type,
147 - unsigned long *obj_size,
148 - union delta_base *delta_base,
149 - unsigned long *next_obj_offset)
155 +static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_base)
156 {
151 - unsigned long pack_limit = pack_size - 20;
152 - unsigned long pos = offset;
153 - unsigned char c;
157 + unsigned char *p, c;
158 unsigned long size, base_offset;
159 unsigned shift;
156 - enum object_type type;
157 - void *data;
160
159 - c = pack_base[pos++];
160 - type = (c >> 4) & 7;
161 + obj->offset = consumed_bytes;
162 +
163 + p = fill(1);
164 + c = *p;
165 + use(1);
166 + obj->type = (c >> 4) & 7;
167 size = (c & 15);
168 shift = 4;
169 while (c & 0x80) {
164 - if (pos >= pack_limit)
165 - bad_object(offset, "object extends past end of pack");
166 - c = pack_base[pos++];
170 + p = fill(1);
171 + c = *p;
172 + use(1);
173 size += (c & 0x7fUL) << shift;
174 shift += 7;
175 }
176 + obj->size = size;
177
171 - switch (type) {
178 + switch (obj->type) {
179 case OBJ_REF_DELTA:
173 - if (pos + 20 >= pack_limit)
174 - bad_object(offset, "object extends past end of pack");
175 - hashcpy(delta_base->sha1, pack_base + pos);
176 - pos += 20;
180 + hashcpy(delta_base->sha1, fill(20));
181 + use(20);
182 break;
183 case OBJ_OFS_DELTA:
184 memset(delta_base, 0, sizeof(*delta_base));
180 - c = pack_base[pos++];
185 + p = fill(1);
186 + c = *p;
187 + use(1);
188 base_offset = c & 127;
189 while (c & 128) {
190 base_offset += 1;
191 if (!base_offset || base_offset & ~(~0UL >> 7))
185 - bad_object(offset, "offset value overflow for delta base object");
186 - if (pos >= pack_limit)
187 - bad_object(offset, "object extends past end of pack");
188 - c = pack_base[pos++];
192 + bad_object(obj->offset, "offset value overflow for delta base object");
193 + p = fill(1);
194 + c = *p;
195 + use(1);
196 base_offset = (base_offset << 7) + (c & 127);
197 }
191 - delta_base->offset = offset - base_offset;
192 - if (delta_base->offset >= offset)
193 - bad_object(offset, "delta base offset is out of bound");
198 + delta_base->offset = obj->offset - base_offset;
199 + if (delta_base->offset >= obj->offset)
200 + bad_object(obj->offset, "delta base offset is out of bound");
201 break;
202 case OBJ_COMMIT:
203 case OBJ_TREE:
@@ -198,13 +205,38 @@ static void *unpack_raw_entry(unsigned long offset,
205 case OBJ_TAG:
206 break;
207 default:
201 - bad_object(offset, "bad object type %d", type);
208 + bad_object(obj->offset, "bad object type %d", obj->type);
209 }
210 + obj->hdr_size = consumed_bytes - obj->offset;
211 +
212 + return unpack_entry_data(obj->offset, obj->size);
213 +}
214 +
215 +static void * get_data_from_pack(struct object_entry *obj)
216 +{
217 + unsigned long from = obj[0].offset + obj[0].hdr_size;
218 + unsigned long len = obj[1].offset - from;
219 + unsigned pg_offset = from % getpagesize();
220 + unsigned char *map, *data;
221 + z_stream stream;
222 + int st;
223
204 - data = unpack_entry_data(offset, &pos, size);
205 - *obj_type = type;
206 - *obj_size = size;
207 - *next_obj_offset = pos;
224 + map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,
225 + input_fd, from - pg_offset);
226 + if (map == MAP_FAILED)
227 + die("cannot mmap packfile '%s': %s", pack_name, strerror(errno));
228 + data = xmalloc(obj->size);
229 + memset(&stream, 0, sizeof(stream));
230 + stream.next_out = data;
231 + stream.avail_out = obj->size;
232 + stream.next_in = map + pg_offset;
233 + stream.avail_in = len;
234 + inflateInit(&stream);
235 + while ((st = inflate(&stream, Z_FINISH)) == Z_OK);
236 + inflateEnd(&stream);
237 + if (st != Z_STREAM_END || stream.total_out != obj->size)
238 + die("serious inflate inconsistency");
239 + munmap(map, len + pg_offset);
240 return data;
241 }
242
@@ -280,15 +312,12 @@ static void resolve_delta(struct delta_entry *delta, void *base_data,
312 unsigned long delta_size;
313 void *result;
314 unsigned long result_size;
283 - enum object_type delta_type;
315 union delta_base delta_base;
285 - unsigned long next_obj_offset;
316 int j, first, last;
317
318 obj->real_type = type;
289 - delta_data = unpack_raw_entry(obj->offset, &delta_type,
290 - &delta_size, &delta_base,
291 - &next_obj_offset);
319 + delta_data = get_data_from_pack(obj);
320 + delta_size = obj->size;
321 result = patch_delta(base_data, base_size, delta_data, delta_size,
322 &result_size);
323 free(delta_data);
@@ -321,13 +350,13 @@ static int compare_delta_entry(const void *a, const void *b)
350 return memcmp(&delta_a->base, &delta_b->base, UNION_BASE_SZ);
351 }
352
324 -static void parse_pack_objects(void)
353 +/* Parse all objects and return the pack content SHA1 hash */
354 +static void parse_pack_objects(unsigned char *sha1)
355 {
356 int i;
327 - unsigned long offset = sizeof(struct pack_header);
357 struct delta_entry *delta = deltas;
358 void *data;
330 - unsigned long data_size;
359 + struct stat st;
360
361 /*
362 * First pass:
@@ -337,19 +366,29 @@ static void parse_pack_objects(void)
366 */
367 for (i = 0; i < nr_objects; i++) {
368 struct object_entry *obj = &objects[i];
340 - obj->offset = offset;
341 - data = unpack_raw_entry(offset, &obj->type, &data_size,
342 - &delta->base, &offset);
369 + data = unpack_raw_entry(obj, &delta->base);
370 obj->real_type = obj->type;
371 if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {
372 nr_deltas++;
373 delta->obj = obj;
374 delta++;
375 } else
349 - sha1_object(data, data_size, obj->type, obj->sha1);
376 + sha1_object(data, obj->size, obj->type, obj->sha1);
377 free(data);
378 }
352 - if (offset != pack_size - 20)
379 + objects[i].offset = consumed_bytes;
380 +
381 + /* Check pack integrity */
382 + SHA1_Update(&input_ctx, input_buffer, input_offset);
383 + SHA1_Final(sha1, &input_ctx);
384 + if (hashcmp(fill(20), sha1))
385 + die("packfile '%s' SHA1 mismatch", pack_name);
386 + use(20);
387 +
388 + /* If input_fd is a file, we should have reached its end now. */
389 + if (fstat(input_fd, &st))
390 + die("cannot fstat packfile '%s': %s", pack_name, strerror(errno));
391 + if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)
392 die("packfile '%s' has junk at the end", pack_name);
393
394 /* Sort deltas by base SHA1/offset for fast searching */
@@ -378,18 +417,17 @@ static void parse_pack_objects(void)
417 ofs = !find_delta_childs(&base, &ofs_first, &ofs_last);
418 if (!ref && !ofs)
419 continue;
381 - data = unpack_raw_entry(obj->offset, &obj->type, &data_size,
382 - &base, &offset);
420 + data = get_data_from_pack(obj);
421 if (ref)
422 for (j = ref_first; j <= ref_last; j++)
423 if (deltas[j].obj->type == OBJ_REF_DELTA)
424 resolve_delta(&deltas[j], data,
387 - data_size, obj->type);
425 + obj->size, obj->type);
426 if (ofs)
427 for (j = ofs_first; j <= ofs_last; j++)
428 if (deltas[j].obj->type == OBJ_OFS_DELTA)
429 resolve_delta(&deltas[j], data,
392 - data_size, obj->type);
430 + obj->size, obj->type);
431 free(data);
432 }
433
@@ -408,6 +446,10 @@ static int sha1_compare(const void *_a, const void *_b)
446 return hashcmp(a->sha1, b->sha1);
447 }
448
449 +/*
450 + * On entry *sha1 contains the pack content SHA1 hash, on exit it is
451 + * the SHA1 hash of sorted object names.
452 + */
453 static void write_index_file(const char *index_name, unsigned char *sha1)
454 {
455 struct sha1file *f;
@@ -467,7 +509,7 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
509 sha1write(f, obj->sha1, 20);
510 SHA1_Update(&ctx, obj->sha1, 20);
511 }
470 - sha1write(f, pack_base + pack_size - 20, 20);
512 + sha1write(f, sha1, 20);
513 sha1close(f, NULL, 1);
514 free(sorted_by_sha);
515 SHA1_Final(sha1, &ctx);
@@ -513,9 +555,9 @@ int main(int argc, char **argv)
555
556 open_pack_file();
557 parse_pack_header();
516 - objects = xcalloc(nr_objects, sizeof(struct object_entry));
558 + objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
559 deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
518 - parse_pack_objects();
560 + parse_pack_objects(sha1);
561 free(deltas);
562 write_index_file(index_name, sha1);
563 free(objects);