add the capability for index-pack to read from a stream
This patch only adds the streaming capability to index-pack. Although the code is different it has the exact same functionality as before to make sure nothing broke. This is in preparation for receiving packs over the net, parse them on the fly, fix them up if they are "thin" packs, and keep the resulting pack instead of exploding it into loose objects. But such functionality should come separately. One immediate advantage of this patch is that index-pack can now deal with packs up to 4GB in size even on 32-bit architectures since the pack is not entirely mmap()'d all at once anymore. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nicolas Pitre committed
Oct 20, 2006 at 14:45 UTC
2d477051ef260aad352d63fc7d9c07e4ebb4359b
1 file changed
+143
-101
index-pack.c
+143
-101
@@ -13,6 +13,8 @@ static const char index_pack_usage[] =
13
struct object_entry
14
{
15
unsigned long offset;
16
+ unsigned long size;
17
+ unsigned int hdr_size;
18
enum object_type type;
19
enum object_type real_type;
20
unsigned char sha1[20];
@@ -36,51 +38,68 @@ struct delta_entry
38
};
39
40
static const char *pack_name;
39
-static unsigned char *pack_base;
40
-static unsigned long pack_size;
41
static struct object_entry *objects;
42
static struct delta_entry *deltas;
43
static int nr_objects;
44
static int nr_deltas;
45
46
-static void open_pack_file(void)
46
+/* We always read in 4kB chunks. */
47
+static unsigned char input_buffer[4096];
48
+static unsigned long input_offset, input_len, consumed_bytes;
49
+static SHA_CTX input_ctx;
50
+static int input_fd;
51
+
52
+/*
53
+ * Make sure at least "min" bytes are available in the buffer, and
54
+ * return the pointer to the buffer.
55
+ */
56
+static void * fill(int min)
57
{
48
- int fd;
49
- struct stat st;
58
+ if (min <= input_len)
59
+ return input_buffer + input_offset;
60
+ if (min > sizeof(input_buffer))
61
+ die("cannot fill %d bytes", min);
62
+ if (input_offset) {
63
+ SHA1_Update(&input_ctx, input_buffer, input_offset);
64
+ memcpy(input_buffer, input_buffer + input_offset, input_len);
65
+ input_offset = 0;
66
+ }
67
+ do {
68
+ int ret = xread(input_fd, input_buffer + input_len,
69
+ sizeof(input_buffer) - input_len);
70
+ if (ret <= 0) {
71
+ if (!ret)
72
+ die("early EOF");
73
+ die("read error on input: %s", strerror(errno));
74
+ }
75
+ input_len += ret;
76
+ } while (input_len < min);
77
+ return input_buffer;
78
+}
79
+
80
+static void use(int bytes)
81
+{
82
+ if (bytes > input_len)
83
+ die("used more bytes than were available");
84
+ input_len -= bytes;
85
+ input_offset += bytes;
86
+ consumed_bytes += bytes;
87
+}
88
51
- fd = open(pack_name, O_RDONLY);
52
- if (fd < 0)
89
+static void open_pack_file(void)
90
+{
91
+ input_fd = open(pack_name, O_RDONLY);
92
+ if (input_fd < 0)
93
die("cannot open packfile '%s': %s", pack_name,
94
strerror(errno));
55
- if (fstat(fd, &st)) {
56
- int err = errno;
57
- close(fd);
58
- die("cannot fstat packfile '%s': %s", pack_name,
59
- strerror(err));
60
- }
61
- pack_size = st.st_size;
62
- pack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
63
- if (pack_base == MAP_FAILED) {
64
- int err = errno;
65
- close(fd);
66
- die("cannot mmap packfile '%s': %s", pack_name,
67
- strerror(err));
68
- }
69
- close(fd);
95
+ SHA1_Init(&input_ctx);
96
}
97
98
static void parse_pack_header(void)
99
{
74
- const struct pack_header *hdr;
75
- unsigned char sha1[20];
76
- SHA_CTX ctx;
77
-
78
- /* Ensure there are enough bytes for the header and final SHA1 */
79
- if (pack_size < sizeof(struct pack_header) + 20)
80
- die("packfile '%s' is too small", pack_name);
100
+ struct pack_header *hdr = fill(sizeof(struct pack_header));
101
102
/* Header consistency check */
83
- hdr = (void *)pack_base;
103
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
104
die("packfile '%s' signature mismatch", pack_name);
105
if (!pack_version_ok(hdr->hdr_version))
@@ -88,13 +107,8 @@ static void parse_pack_header(void)
107
pack_name, ntohl(hdr->hdr_version));
108
109
nr_objects = ntohl(hdr->hdr_entries);
91
-
92
- /* Check packfile integrity */
93
- SHA1_Init(&ctx);
94
- SHA1_Update(&ctx, pack_base, pack_size - 20);
95
- SHA1_Final(sha1, &ctx);
96
- if (hashcmp(sha1, pack_base + pack_size - 20))
97
- die("packfile '%s' SHA1 mismatch", pack_name);
110
+ use(sizeof(struct pack_header));
111
+ /*fprintf(stderr, "Indexing %d objects\n", nr_objects);*/
112
}
113
114
static void bad_object(unsigned long offset, const char *format,
@@ -112,85 +126,78 @@ static void bad_object(unsigned long offset, const char *format, ...)
126
pack_name, offset, buf);
127
}
128
115
-static void *unpack_entry_data(unsigned long offset,
116
- unsigned long *current_pos, unsigned long size)
129
+static void *unpack_entry_data(unsigned long offset, unsigned long size)
130
{
118
- unsigned long pack_limit = pack_size - 20;
119
- unsigned long pos = *current_pos;
131
z_stream stream;
132
void *buf = xmalloc(size);
133
134
memset(&stream, 0, sizeof(stream));
135
stream.next_out = buf;
136
stream.avail_out = size;
126
- stream.next_in = pack_base + pos;
127
- stream.avail_in = pack_limit - pos;
137
+ stream.next_in = fill(1);
138
+ stream.avail_in = input_len;
139
inflateInit(&stream);
140
141
for (;;) {
142
int ret = inflate(&stream, 0);
132
- if (ret == Z_STREAM_END)
143
+ use(input_len - stream.avail_in);
144
+ if (stream.total_out == size && ret == Z_STREAM_END)
145
break;
146
if (ret != Z_OK)
147
bad_object(offset, "inflate returned %d", ret);
148
+ stream.next_in = fill(1);
149
+ stream.avail_in = input_len;
150
}
151
inflateEnd(&stream);
138
- if (stream.total_out != size)
139
- bad_object(offset, "size mismatch (expected %lu, got %lu)",
140
- size, stream.total_out);
141
- *current_pos = pack_limit - stream.avail_in;
152
return buf;
153
}
154
145
-static void *unpack_raw_entry(unsigned long offset,
146
- enum object_type *obj_type,
147
- unsigned long *obj_size,
148
- union delta_base *delta_base,
149
- unsigned long *next_obj_offset)
155
+static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_base)
156
{
151
- unsigned long pack_limit = pack_size - 20;
152
- unsigned long pos = offset;
153
- unsigned char c;
157
+ unsigned char *p, c;
158
unsigned long size, base_offset;
159
unsigned shift;
156
- enum object_type type;
157
- void *data;
160
159
- c = pack_base[pos++];
160
- type = (c >> 4) & 7;
161
+ obj->offset = consumed_bytes;
162
+
163
+ p = fill(1);
164
+ c = *p;
165
+ use(1);
166
+ obj->type = (c >> 4) & 7;
167
size = (c & 15);
168
shift = 4;
169
while (c & 0x80) {
164
- if (pos >= pack_limit)
165
- bad_object(offset, "object extends past end of pack");
166
- c = pack_base[pos++];
170
+ p = fill(1);
171
+ c = *p;
172
+ use(1);
173
size += (c & 0x7fUL) << shift;
174
shift += 7;
175
}
176
+ obj->size = size;
177
171
- switch (type) {
178
+ switch (obj->type) {
179
case OBJ_REF_DELTA:
173
- if (pos + 20 >= pack_limit)
174
- bad_object(offset, "object extends past end of pack");
175
- hashcpy(delta_base->sha1, pack_base + pos);
176
- pos += 20;
180
+ hashcpy(delta_base->sha1, fill(20));
181
+ use(20);
182
break;
183
case OBJ_OFS_DELTA:
184
memset(delta_base, 0, sizeof(*delta_base));
180
- c = pack_base[pos++];
185
+ p = fill(1);
186
+ c = *p;
187
+ use(1);
188
base_offset = c & 127;
189
while (c & 128) {
190
base_offset += 1;
191
if (!base_offset || base_offset & ~(~0UL >> 7))
185
- bad_object(offset, "offset value overflow for delta base object");
186
- if (pos >= pack_limit)
187
- bad_object(offset, "object extends past end of pack");
188
- c = pack_base[pos++];
192
+ bad_object(obj->offset, "offset value overflow for delta base object");
193
+ p = fill(1);
194
+ c = *p;
195
+ use(1);
196
base_offset = (base_offset << 7) + (c & 127);
197
}
191
- delta_base->offset = offset - base_offset;
192
- if (delta_base->offset >= offset)
193
- bad_object(offset, "delta base offset is out of bound");
198
+ delta_base->offset = obj->offset - base_offset;
199
+ if (delta_base->offset >= obj->offset)
200
+ bad_object(obj->offset, "delta base offset is out of bound");
201
break;
202
case OBJ_COMMIT:
203
case OBJ_TREE:
@@ -198,13 +205,38 @@ static void *unpack_raw_entry(unsigned long offset,
205
case OBJ_TAG:
206
break;
207
default:
201
- bad_object(offset, "bad object type %d", type);
208
+ bad_object(obj->offset, "bad object type %d", obj->type);
209
}
210
+ obj->hdr_size = consumed_bytes - obj->offset;
211
+
212
+ return unpack_entry_data(obj->offset, obj->size);
213
+}
214
+
215
+static void * get_data_from_pack(struct object_entry *obj)
216
+{
217
+ unsigned long from = obj[0].offset + obj[0].hdr_size;
218
+ unsigned long len = obj[1].offset - from;
219
+ unsigned pg_offset = from % getpagesize();
220
+ unsigned char *map, *data;
221
+ z_stream stream;
222
+ int st;
223
204
- data = unpack_entry_data(offset, &pos, size);
205
- *obj_type = type;
206
- *obj_size = size;
207
- *next_obj_offset = pos;
224
+ map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,
225
+ input_fd, from - pg_offset);
226
+ if (map == MAP_FAILED)
227
+ die("cannot mmap packfile '%s': %s", pack_name, strerror(errno));
228
+ data = xmalloc(obj->size);
229
+ memset(&stream, 0, sizeof(stream));
230
+ stream.next_out = data;
231
+ stream.avail_out = obj->size;
232
+ stream.next_in = map + pg_offset;
233
+ stream.avail_in = len;
234
+ inflateInit(&stream);
235
+ while ((st = inflate(&stream, Z_FINISH)) == Z_OK);
236
+ inflateEnd(&stream);
237
+ if (st != Z_STREAM_END || stream.total_out != obj->size)
238
+ die("serious inflate inconsistency");
239
+ munmap(map, len + pg_offset);
240
return data;
241
}
242
@@ -280,15 +312,12 @@ static void resolve_delta(struct delta_entry *delta, void *base_data,
312
unsigned long delta_size;
313
void *result;
314
unsigned long result_size;
283
- enum object_type delta_type;
315
union delta_base delta_base;
285
- unsigned long next_obj_offset;
316
int j, first, last;
317
318
obj->real_type = type;
289
- delta_data = unpack_raw_entry(obj->offset, &delta_type,
290
- &delta_size, &delta_base,
291
- &next_obj_offset);
319
+ delta_data = get_data_from_pack(obj);
320
+ delta_size = obj->size;
321
result = patch_delta(base_data, base_size, delta_data, delta_size,
322
&result_size);
323
free(delta_data);
@@ -321,13 +350,13 @@ static int compare_delta_entry(const void *a, const void *b)
350
return memcmp(&delta_a->base, &delta_b->base, UNION_BASE_SZ);
351
}
352
324
-static void parse_pack_objects(void)
353
+/* Parse all objects and return the pack content SHA1 hash */
354
+static void parse_pack_objects(unsigned char *sha1)
355
{
356
int i;
327
- unsigned long offset = sizeof(struct pack_header);
357
struct delta_entry *delta = deltas;
358
void *data;
330
- unsigned long data_size;
359
+ struct stat st;
360
361
/*
362
* First pass:
@@ -337,19 +366,29 @@ static void parse_pack_objects(void)
366
*/
367
for (i = 0; i < nr_objects; i++) {
368
struct object_entry *obj = &objects[i];
340
- obj->offset = offset;
341
- data = unpack_raw_entry(offset, &obj->type, &data_size,
342
- &delta->base, &offset);
369
+ data = unpack_raw_entry(obj, &delta->base);
370
obj->real_type = obj->type;
371
if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {
372
nr_deltas++;
373
delta->obj = obj;
374
delta++;
375
} else
349
- sha1_object(data, data_size, obj->type, obj->sha1);
376
+ sha1_object(data, obj->size, obj->type, obj->sha1);
377
free(data);
378
}
352
- if (offset != pack_size - 20)
379
+ objects[i].offset = consumed_bytes;
380
+
381
+ /* Check pack integrity */
382
+ SHA1_Update(&input_ctx, input_buffer, input_offset);
383
+ SHA1_Final(sha1, &input_ctx);
384
+ if (hashcmp(fill(20), sha1))
385
+ die("packfile '%s' SHA1 mismatch", pack_name);
386
+ use(20);
387
+
388
+ /* If input_fd is a file, we should have reached its end now. */
389
+ if (fstat(input_fd, &st))
390
+ die("cannot fstat packfile '%s': %s", pack_name, strerror(errno));
391
+ if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)
392
die("packfile '%s' has junk at the end", pack_name);
393
394
/* Sort deltas by base SHA1/offset for fast searching */
@@ -378,18 +417,17 @@ static void parse_pack_objects(void)
417
ofs = !find_delta_childs(&base, &ofs_first, &ofs_last);
418
if (!ref && !ofs)
419
continue;
381
- data = unpack_raw_entry(obj->offset, &obj->type, &data_size,
382
- &base, &offset);
420
+ data = get_data_from_pack(obj);
421
if (ref)
422
for (j = ref_first; j <= ref_last; j++)
423
if (deltas[j].obj->type == OBJ_REF_DELTA)
424
resolve_delta(&deltas[j], data,
387
- data_size, obj->type);
425
+ obj->size, obj->type);
426
if (ofs)
427
for (j = ofs_first; j <= ofs_last; j++)
428
if (deltas[j].obj->type == OBJ_OFS_DELTA)
429
resolve_delta(&deltas[j], data,
392
- data_size, obj->type);
430
+ obj->size, obj->type);
431
free(data);
432
}
433
@@ -408,6 +446,10 @@ static int sha1_compare(const void *_a, const void *_b)
446
return hashcmp(a->sha1, b->sha1);
447
}
448
449
+/*
450
+ * On entry *sha1 contains the pack content SHA1 hash, on exit it is
451
+ * the SHA1 hash of sorted object names.
452
+ */
453
static void write_index_file(const char *index_name, unsigned char *sha1)
454
{
455
struct sha1file *f;
@@ -467,7 +509,7 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
509
sha1write(f, obj->sha1, 20);
510
SHA1_Update(&ctx, obj->sha1, 20);
511
}
470
- sha1write(f, pack_base + pack_size - 20, 20);
512
+ sha1write(f, sha1, 20);
513
sha1close(f, NULL, 1);
514
free(sorted_by_sha);
515
SHA1_Final(sha1, &ctx);
@@ -513,9 +555,9 @@ int main(int argc, char **argv)
555
556
open_pack_file();
557
parse_pack_header();
516
- objects = xcalloc(nr_objects, sizeof(struct object_entry));
558
+ objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
559
deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
518
- parse_pack_objects();
560
+ parse_pack_objects(sha1);
561
free(deltas);
562
write_index_file(index_name, sha1);
563
free(objects);