@matheusbsbs / matheus-bagliano / commits / 9cf6d3357a
Fork 0

Add git-index-pack utility

git-index-pack builds a pack index file for an existing packed archive. With this utility a packed archive which was transferred without the corresponding pack index can be added to objects/pack/ without repacking. Signed-off-by: Sergey Vlasov <vsu@altlinux.ru> Signed-off-by: Junio C Hamano <junkio@cox.net>

Sergey Vlasov committed Oct 12, 2005 at 12:01 UTC 9cf6d3357aaaaa89dd86cc156221b7b604e9358c
5 files changed +517 -1
Documentation/git-index-pack.txt new
+44
@@ -0,0 +1,44 @@
1 +git-index-pack(1)
2 +=================
3 +
4 +NAME
5 +----
6 +git-index-pack - Build pack index file for an existing packed archive
7 +
8 +
9 +SYNOPSIS
10 +--------
11 +'git-index-pack' [-o <index-file>] <pack-file>
12 +
13 +
14 +DESCRIPTION
15 +-----------
16 +Reads a packed archive (.pack) from the specified file, and
17 +builds a pack index file (.idx) for it. The packed archive
18 +together with the pack index can then be placed in the
19 +objects/pack/ directory of a git repository.
20 +
21 +
22 +OPTIONS
23 +-------
24 +-o <index-file>::
25 + Write the generated pack index into the specified
26 + file. Without this option the name of pack index
27 + file is constructed from the name of packed archive
28 + file by replacing .pack with .idx (and the program
29 + fails if the name of packed archive does not end
30 + with .pack).
31 +
32 +
33 +Author
34 +------
35 +Written by Sergey Vlasov <vsu@altlinux.ru>
36 +
37 +Documentation
38 +-------------
39 +Documentation by Sergey Vlasov
40 +
41 +GIT
42 +---
43 +Part of the gitlink:git[7] suite
44 +
Documentation/git.txt
+3
@@ -68,6 +68,9 @@ gitlink:git-commit-tree[1]::
68 gitlink:git-hash-object[1]::
69 Computes the object ID from a file.
70
71 +gitlink:git-index-pack.html[1]::
72 + Build pack index file for an existing packed archive.
73 +
74 gitlink:git-init-db[1]::
75 Creates an empty git object database
76
Makefile
+1 -1
@@ -110,7 +110,7 @@ PROGRAMS = \
110 git-convert-objects$X git-diff-files$X \
111 git-diff-index$X git-diff-stages$X \
112 git-diff-tree$X git-fetch-pack$X git-fsck-objects$X \
113 - git-hash-object$X git-init-db$X \
113 + git-hash-object$X git-index-pack$X git-init-db$X \
114 git-local-fetch$X git-ls-files$X git-ls-tree$X git-merge-base$X \
115 git-merge-index$X git-mktag$X git-pack-objects$X git-patch-id$X \
116 git-peek-remote$X git-prune-packed$X git-read-tree$X \
index-pack.c new
+451
@@ -0,0 +1,451 @@
1 +#include "cache.h"
2 +#include "delta.h"
3 +#include "pack.h"
4 +#include "csum-file.h"
5 +
6 +static const char index_pack_usage[] =
7 +"git-index-pack [-o index-file] pack-file";
8 +
9 +struct object_entry
10 +{
11 + unsigned long offset;
12 + enum object_type type;
13 + enum object_type real_type;
14 + unsigned char sha1[20];
15 +};
16 +
17 +struct delta_entry
18 +{
19 + struct object_entry *obj;
20 + unsigned char base_sha1[20];
21 +};
22 +
23 +static const char *pack_name;
24 +static unsigned char *pack_base;
25 +static unsigned long pack_size;
26 +static struct object_entry *objects;
27 +static struct delta_entry *deltas;
28 +static int nr_objects;
29 +static int nr_deltas;
30 +
31 +static void open_pack_file(void)
32 +{
33 + int fd;
34 + struct stat st;
35 +
36 + fd = open(pack_name, O_RDONLY);
37 + if (fd < 0)
38 + die("cannot open packfile '%s': %s", pack_name,
39 + strerror(errno));
40 + if (fstat(fd, &st)) {
41 + int err = errno;
42 + close(fd);
43 + die("cannot fstat packfile '%s': %s", pack_name,
44 + strerror(err));
45 + }
46 + pack_size = st.st_size;
47 + pack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
48 + if (pack_base == MAP_FAILED) {
49 + int err = errno;
50 + close(fd);
51 + die("cannot mmap packfile '%s': %s", pack_name,
52 + strerror(err));
53 + }
54 + close(fd);
55 +}
56 +
57 +static void parse_pack_header(void)
58 +{
59 + const struct pack_header *hdr;
60 + unsigned char sha1[20];
61 + SHA_CTX ctx;
62 +
63 + /* Ensure there are enough bytes for the header and final SHA1 */
64 + if (pack_size < sizeof(struct pack_header) + 20)
65 + die("packfile '%s' is too small", pack_name);
66 +
67 + /* Header consistency check */
68 + hdr = (void *)pack_base;
69 + if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
70 + die("packfile '%s' signature mismatch", pack_name);
71 + if (hdr->hdr_version != htonl(PACK_VERSION))
72 + die("packfile '%s' version %d different from ours %d",
73 + pack_name, ntohl(hdr->hdr_version), PACK_VERSION);
74 +
75 + nr_objects = ntohl(hdr->hdr_entries);
76 +
77 + /* Check packfile integrity */
78 + SHA1_Init(&ctx);
79 + SHA1_Update(&ctx, pack_base, pack_size - 20);
80 + SHA1_Final(sha1, &ctx);
81 + if (memcmp(sha1, pack_base + pack_size - 20, 20))
82 + die("packfile '%s' SHA1 mismatch", pack_name);
83 +}
84 +
85 +static void bad_object(unsigned long offset, const char *format,
86 + ...) NORETURN __attribute__((format (printf, 2, 3)));
87 +
88 +static void bad_object(unsigned long offset, const char *format, ...)
89 +{
90 + va_list params;
91 + char buf[1024];
92 +
93 + va_start(params, format);
94 + vsnprintf(buf, sizeof(buf), format, params);
95 + va_end(params);
96 + die("packfile '%s': bad object at offset %lu: %s",
97 + pack_name, offset, buf);
98 +}
99 +
100 +static void *unpack_entry_data(unsigned long offset,
101 + unsigned long *current_pos, unsigned long size)
102 +{
103 + unsigned long pack_limit = pack_size - 20;
104 + unsigned long pos = *current_pos;
105 + z_stream stream;
106 + void *buf = xmalloc(size);
107 +
108 + memset(&stream, 0, sizeof(stream));
109 + stream.next_out = buf;
110 + stream.avail_out = size;
111 + stream.next_in = pack_base + pos;
112 + stream.avail_in = pack_limit - pos;
113 + inflateInit(&stream);
114 +
115 + for (;;) {
116 + int ret = inflate(&stream, 0);
117 + if (ret == Z_STREAM_END)
118 + break;
119 + if (ret != Z_OK)
120 + bad_object(offset, "inflate returned %d", ret);
121 + }
122 + inflateEnd(&stream);
123 + if (stream.total_out != size)
124 + bad_object(offset, "size mismatch (expected %lu, got %lu)",
125 + size, stream.total_out);
126 + *current_pos = pack_limit - stream.avail_in;
127 + return buf;
128 +}
129 +
130 +static void *unpack_raw_entry(unsigned long offset,
131 + enum object_type *obj_type,
132 + unsigned long *obj_size,
133 + unsigned char *delta_base,
134 + unsigned long *next_obj_offset)
135 +{
136 + unsigned long pack_limit = pack_size - 20;
137 + unsigned long pos = offset;
138 + unsigned char c;
139 + unsigned long size;
140 + unsigned shift;
141 + enum object_type type;
142 + void *data;
143 +
144 + c = pack_base[pos++];
145 + type = (c >> 4) & 7;
146 + size = (c & 15);
147 + shift = 4;
148 + while (c & 0x80) {
149 + if (pos >= pack_limit)
150 + bad_object(offset, "object extends past end of pack");
151 + c = pack_base[pos++];
152 + size += (c & 0x7fUL) << shift;
153 + shift += 7;
154 + }
155 +
156 + switch (type) {
157 + case OBJ_DELTA:
158 + if (pos + 20 >= pack_limit)
159 + bad_object(offset, "object extends past end of pack");
160 + memcpy(delta_base, pack_base + pos, 20);
161 + pos += 20;
162 + /* fallthru */
163 + case OBJ_COMMIT:
164 + case OBJ_TREE:
165 + case OBJ_BLOB:
166 + case OBJ_TAG:
167 + data = unpack_entry_data(offset, &pos, size);
168 + break;
169 + default:
170 + bad_object(offset, "bad object type %d", type);
171 + }
172 +
173 + *obj_type = type;
174 + *obj_size = size;
175 + *next_obj_offset = pos;
176 + return data;
177 +}
178 +
179 +static int find_delta(const unsigned char *base_sha1)
180 +{
181 + int first = 0, last = nr_deltas;
182 +
183 + while (first < last) {
184 + int next = (first + last) / 2;
185 + struct delta_entry *delta = &deltas[next];
186 + int cmp;
187 +
188 + cmp = memcmp(base_sha1, delta->base_sha1, 20);
189 + if (!cmp)
190 + return next;
191 + if (cmp < 0) {
192 + last = next;
193 + continue;
194 + }
195 + first = next+1;
196 + }
197 + return -first-1;
198 +}
199 +
200 +static int find_deltas_based_on_sha1(const unsigned char *base_sha1,
201 + int *first_index, int *last_index)
202 +{
203 + int first = find_delta(base_sha1);
204 + int last = first;
205 + int end = nr_deltas - 1;
206 +
207 + if (first < 0)
208 + return -1;
209 + while (first > 0 && !memcmp(deltas[first-1].base_sha1, base_sha1, 20))
210 + --first;
211 + while (last < end && !memcmp(deltas[last+1].base_sha1, base_sha1, 20))
212 + ++last;
213 + *first_index = first;
214 + *last_index = last;
215 + return 0;
216 +}
217 +
218 +static void sha1_object(const void *data, unsigned long size,
219 + enum object_type type, unsigned char *sha1)
220 +{
221 + SHA_CTX ctx;
222 + char header[50];
223 + int header_size;
224 + const char *type_str;
225 +
226 + switch (type) {
227 + case OBJ_COMMIT: type_str = "commit"; break;
228 + case OBJ_TREE: type_str = "tree"; break;
229 + case OBJ_BLOB: type_str = "blob"; break;
230 + case OBJ_TAG: type_str = "tag"; break;
231 + default:
232 + die("bad type %d", type);
233 + }
234 +
235 + header_size = sprintf(header, "%s %lu", type_str, size) + 1;
236 +
237 + SHA1_Init(&ctx);
238 + SHA1_Update(&ctx, header, header_size);
239 + SHA1_Update(&ctx, data, size);
240 + SHA1_Final(sha1, &ctx);
241 +}
242 +
243 +static void resolve_delta(struct delta_entry *delta, void *base_data,
244 + unsigned long base_size, enum object_type type)
245 +{
246 + struct object_entry *obj = delta->obj;
247 + void *delta_data;
248 + unsigned long delta_size;
249 + void *result;
250 + unsigned long result_size;
251 + enum object_type delta_type;
252 + unsigned char base_sha1[20];
253 + unsigned long next_obj_offset;
254 + int j, first, last;
255 +
256 + obj->real_type = type;
257 + delta_data = unpack_raw_entry(obj->offset, &delta_type,
258 + &delta_size, base_sha1,
259 + &next_obj_offset);
260 + result = patch_delta(base_data, base_size, delta_data, delta_size,
261 + &result_size);
262 + free(delta_data);
263 + if (!result)
264 + bad_object(obj->offset, "failed to apply delta");
265 + sha1_object(result, result_size, type, obj->sha1);
266 + if (!find_deltas_based_on_sha1(obj->sha1, &first, &last)) {
267 + for (j = first; j <= last; j++)
268 + resolve_delta(&deltas[j], result, result_size, type);
269 + }
270 + free(result);
271 +}
272 +
273 +static int compare_delta_entry(const void *a, const void *b)
274 +{
275 + const struct delta_entry *delta_a = a;
276 + const struct delta_entry *delta_b = b;
277 + return memcmp(delta_a->base_sha1, delta_b->base_sha1, 20);
278 +}
279 +
280 +static void parse_pack_objects(void)
281 +{
282 + int i;
283 + unsigned long offset = sizeof(struct pack_header);
284 + unsigned char base_sha1[20];
285 + void *data;
286 + unsigned long data_size;
287 +
288 + /*
289 + * First pass:
290 + * - find locations of all objects;
291 + * - calculate SHA1 of all non-delta objects;
292 + * - remember base SHA1 for all deltas.
293 + */
294 + for (i = 0; i < nr_objects; i++) {
295 + struct object_entry *obj = &objects[i];
296 + obj->offset = offset;
297 + data = unpack_raw_entry(offset, &obj->type, &data_size,
298 + base_sha1, &offset);
299 + obj->real_type = obj->type;
300 + if (obj->type == OBJ_DELTA) {
301 + struct delta_entry *delta = &deltas[nr_deltas++];
302 + delta->obj = obj;
303 + memcpy(delta->base_sha1, base_sha1, 20);
304 + } else
305 + sha1_object(data, data_size, obj->type, obj->sha1);
306 + free(data);
307 + }
308 + if (offset != pack_size - 20)
309 + die("packfile '%s' has junk at the end", pack_name);
310 +
311 + /* Sort deltas by base SHA1 for fast searching */
312 + qsort(deltas, nr_deltas, sizeof(struct delta_entry),
313 + compare_delta_entry);
314 +
315 + /*
316 + * Second pass:
317 + * - for all non-delta objects, look if it is used as a base for
318 + * deltas;
319 + * - if used as a base, uncompress the object and apply all deltas,
320 + * recursively checking if the resulting object is used as a base
321 + * for some more deltas.
322 + */
323 + for (i = 0; i < nr_objects; i++) {
324 + struct object_entry *obj = &objects[i];
325 + int j, first, last;
326 +
327 + if (obj->type == OBJ_DELTA)
328 + continue;
329 + if (find_deltas_based_on_sha1(obj->sha1, &first, &last))
330 + continue;
331 + data = unpack_raw_entry(obj->offset, &obj->type, &data_size,
332 + base_sha1, &offset);
333 + for (j = first; j <= last; j++)
334 + resolve_delta(&deltas[j], data, data_size, obj->type);
335 + free(data);
336 + }
337 +
338 + /* Check for unresolved deltas */
339 + for (i = 0; i < nr_deltas; i++) {
340 + if (deltas[i].obj->real_type == OBJ_DELTA)
341 + die("packfile '%s' has unresolved deltas", pack_name);
342 + }
343 +}
344 +
345 +static int sha1_compare(const void *_a, const void *_b)
346 +{
347 + struct object_entry *a = *(struct object_entry **)_a;
348 + struct object_entry *b = *(struct object_entry **)_b;
349 + return memcmp(a->sha1, b->sha1, 20);
350 +}
351 +
352 +static void write_index_file(const char *index_name)
353 +{
354 + struct sha1file *f;
355 + struct object_entry **sorted_by_sha =
356 + xcalloc(nr_objects, sizeof(struct object_entry *));
357 + struct object_entry **list = sorted_by_sha;
358 + struct object_entry **last = sorted_by_sha + nr_objects;
359 + unsigned int array[256];
360 + int i;
361 +
362 + for (i = 0; i < nr_objects; ++i)
363 + sorted_by_sha[i] = &objects[i];
364 + qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
365 + sha1_compare);
366 +
367 + unlink(index_name);
368 + f = sha1create("%s", index_name);
369 +
370 + /*
371 + * Write the first-level table (the list is sorted,
372 + * but we use a 256-entry lookup to be able to avoid
373 + * having to do eight extra binary search iterations).
374 + */
375 + for (i = 0; i < 256; i++) {
376 + struct object_entry **next = list;
377 + while (next < last) {
378 + struct object_entry *obj = *next;
379 + if (obj->sha1[0] != i)
380 + break;
381 + next++;
382 + }
383 + array[i] = htonl(next - sorted_by_sha);
384 + list = next;
385 + }
386 + sha1write(f, array, 256 * sizeof(int));
387 +
388 + /*
389 + * Write the actual SHA1 entries..
390 + */
391 + list = sorted_by_sha;
392 + for (i = 0; i < nr_objects; i++) {
393 + struct object_entry *obj = *list++;
394 + unsigned int offset = htonl(obj->offset);
395 + sha1write(f, &offset, 4);
396 + sha1write(f, obj->sha1, 20);
397 + }
398 + sha1write(f, pack_base + pack_size - 20, 20);
399 + sha1close(f, NULL, 1);
400 + free(sorted_by_sha);
401 +}
402 +
403 +int main(int argc, char **argv)
404 +{
405 + int i;
406 + char *index_name = NULL;
407 + char *index_name_buf = NULL;
408 +
409 + for (i = 1; i < argc; i++) {
410 + const char *arg = argv[i];
411 +
412 + if (*arg == '-') {
413 + if (!strcmp(arg, "-o")) {
414 + if (index_name || (i+1) >= argc)
415 + usage(index_pack_usage);
416 + index_name = argv[++i];
417 + } else
418 + usage(index_pack_usage);
419 + continue;
420 + }
421 +
422 + if (pack_name)
423 + usage(index_pack_usage);
424 + pack_name = arg;
425 + }
426 +
427 + if (!pack_name)
428 + usage(index_pack_usage);
429 + if (!index_name) {
430 + int len = strlen(pack_name);
431 + if (len < 5 || strcmp(pack_name + len - 5, ".pack"))
432 + die("packfile name '%s' does not end with '.pack'",
433 + pack_name);
434 + index_name_buf = xmalloc(len - 1);
435 + memcpy(index_name_buf, pack_name, len - 5);
436 + strcpy(index_name_buf + len - 5, ".idx");
437 + index_name = index_name_buf;
438 + }
439 +
440 + open_pack_file();
441 + parse_pack_header();
442 + objects = xcalloc(nr_objects, sizeof(struct object_entry));
443 + deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
444 + parse_pack_objects();
445 + free(deltas);
446 + write_index_file(index_name);
447 + free(objects);
448 + free(index_name_buf);
449 +
450 + return 0;
451 +}
t/t5300-pack-object.sh
+18
@@ -165,4 +165,22 @@ test_expect_success \
165
166 :'
167
168 +test_expect_success \
169 + 'build pack index for an existing pack' \
170 + 'cp test-1-${packname_1}.pack test-3.pack &&
171 + git-index-pack -o tmp.idx test-3.pack &&
172 + cmp tmp.idx test-1-${packname_1}.idx &&
173 +
174 + git-index-pack test-3.pack &&
175 + cmp test-3.idx test-1-${packname_1}.idx &&
176 +
177 + cp test-2-${packname_2}.pack test-3.pack &&
178 + git-index-pack -o tmp.idx test-2-${packname_2}.pack &&
179 + cmp tmp.idx test-2-${packname_2}.idx &&
180 +
181 + git-index-pack test-3.pack &&
182 + cmp test-3.idx test-2-${packname_2}.idx &&
183 +
184 + :'
185 +
186 test_done