Add git-index-pack utility
git-index-pack builds a pack index file for an existing packed archive. With this utility a packed archive which was transferred without the corresponding pack index can be added to objects/pack/ without repacking. Signed-off-by: Sergey Vlasov <vsu@altlinux.ru> Signed-off-by: Junio C Hamano <junkio@cox.net>
Sergey Vlasov committed
Oct 12, 2005 at 12:01 UTC
9cf6d3357aaaaa89dd86cc156221b7b604e9358c
5 files changed
+517
-1
Documentation/git-index-pack.txt
new
+44
@@ -0,0 +1,44 @@
1
+git-index-pack(1)
2
+=================
3
+
4
+NAME
5
+----
6
+git-index-pack - Build pack index file for an existing packed archive
7
+
8
+
9
+SYNOPSIS
10
+--------
11
+'git-index-pack' [-o <index-file>] <pack-file>
12
+
13
+
14
+DESCRIPTION
15
+-----------
16
+Reads a packed archive (.pack) from the specified file, and
17
+builds a pack index file (.idx) for it. The packed archive
18
+together with the pack index can then be placed in the
19
+objects/pack/ directory of a git repository.
20
+
21
+
22
+OPTIONS
23
+-------
24
+-o <index-file>::
25
+ Write the generated pack index into the specified
26
+ file. Without this option the name of pack index
27
+ file is constructed from the name of packed archive
28
+ file by replacing .pack with .idx (and the program
29
+ fails if the name of packed archive does not end
30
+ with .pack).
31
+
32
+
33
+Author
34
+------
35
+Written by Sergey Vlasov <vsu@altlinux.ru>
36
+
37
+Documentation
38
+-------------
39
+Documentation by Sergey Vlasov
40
+
41
+GIT
42
+---
43
+Part of the gitlink:git[7] suite
44
+
Documentation/git.txt
+3
@@ -68,6 +68,9 @@ gitlink:git-commit-tree[1]::
68
gitlink:git-hash-object[1]::
69
Computes the object ID from a file.
70
71
+gitlink:git-index-pack.html[1]::
72
+ Build pack index file for an existing packed archive.
73
+
74
gitlink:git-init-db[1]::
75
Creates an empty git object database
76
Makefile
+1
-1
@@ -110,7 +110,7 @@ PROGRAMS = \
110
git-convert-objects$X git-diff-files$X \
111
git-diff-index$X git-diff-stages$X \
112
git-diff-tree$X git-fetch-pack$X git-fsck-objects$X \
113
- git-hash-object$X git-init-db$X \
113
+ git-hash-object$X git-index-pack$X git-init-db$X \
114
git-local-fetch$X git-ls-files$X git-ls-tree$X git-merge-base$X \
115
git-merge-index$X git-mktag$X git-pack-objects$X git-patch-id$X \
116
git-peek-remote$X git-prune-packed$X git-read-tree$X \
index-pack.c
new
+451
@@ -0,0 +1,451 @@
1
+#include "cache.h"
2
+#include "delta.h"
3
+#include "pack.h"
4
+#include "csum-file.h"
5
+
6
+static const char index_pack_usage[] =
7
+"git-index-pack [-o index-file] pack-file";
8
+
9
+struct object_entry
10
+{
11
+ unsigned long offset;
12
+ enum object_type type;
13
+ enum object_type real_type;
14
+ unsigned char sha1[20];
15
+};
16
+
17
+struct delta_entry
18
+{
19
+ struct object_entry *obj;
20
+ unsigned char base_sha1[20];
21
+};
22
+
23
+static const char *pack_name;
24
+static unsigned char *pack_base;
25
+static unsigned long pack_size;
26
+static struct object_entry *objects;
27
+static struct delta_entry *deltas;
28
+static int nr_objects;
29
+static int nr_deltas;
30
+
31
+static void open_pack_file(void)
32
+{
33
+ int fd;
34
+ struct stat st;
35
+
36
+ fd = open(pack_name, O_RDONLY);
37
+ if (fd < 0)
38
+ die("cannot open packfile '%s': %s", pack_name,
39
+ strerror(errno));
40
+ if (fstat(fd, &st)) {
41
+ int err = errno;
42
+ close(fd);
43
+ die("cannot fstat packfile '%s': %s", pack_name,
44
+ strerror(err));
45
+ }
46
+ pack_size = st.st_size;
47
+ pack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
48
+ if (pack_base == MAP_FAILED) {
49
+ int err = errno;
50
+ close(fd);
51
+ die("cannot mmap packfile '%s': %s", pack_name,
52
+ strerror(err));
53
+ }
54
+ close(fd);
55
+}
56
+
57
+static void parse_pack_header(void)
58
+{
59
+ const struct pack_header *hdr;
60
+ unsigned char sha1[20];
61
+ SHA_CTX ctx;
62
+
63
+ /* Ensure there are enough bytes for the header and final SHA1 */
64
+ if (pack_size < sizeof(struct pack_header) + 20)
65
+ die("packfile '%s' is too small", pack_name);
66
+
67
+ /* Header consistency check */
68
+ hdr = (void *)pack_base;
69
+ if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
70
+ die("packfile '%s' signature mismatch", pack_name);
71
+ if (hdr->hdr_version != htonl(PACK_VERSION))
72
+ die("packfile '%s' version %d different from ours %d",
73
+ pack_name, ntohl(hdr->hdr_version), PACK_VERSION);
74
+
75
+ nr_objects = ntohl(hdr->hdr_entries);
76
+
77
+ /* Check packfile integrity */
78
+ SHA1_Init(&ctx);
79
+ SHA1_Update(&ctx, pack_base, pack_size - 20);
80
+ SHA1_Final(sha1, &ctx);
81
+ if (memcmp(sha1, pack_base + pack_size - 20, 20))
82
+ die("packfile '%s' SHA1 mismatch", pack_name);
83
+}
84
+
85
+static void bad_object(unsigned long offset, const char *format,
86
+ ...) NORETURN __attribute__((format (printf, 2, 3)));
87
+
88
+static void bad_object(unsigned long offset, const char *format, ...)
89
+{
90
+ va_list params;
91
+ char buf[1024];
92
+
93
+ va_start(params, format);
94
+ vsnprintf(buf, sizeof(buf), format, params);
95
+ va_end(params);
96
+ die("packfile '%s': bad object at offset %lu: %s",
97
+ pack_name, offset, buf);
98
+}
99
+
100
+static void *unpack_entry_data(unsigned long offset,
101
+ unsigned long *current_pos, unsigned long size)
102
+{
103
+ unsigned long pack_limit = pack_size - 20;
104
+ unsigned long pos = *current_pos;
105
+ z_stream stream;
106
+ void *buf = xmalloc(size);
107
+
108
+ memset(&stream, 0, sizeof(stream));
109
+ stream.next_out = buf;
110
+ stream.avail_out = size;
111
+ stream.next_in = pack_base + pos;
112
+ stream.avail_in = pack_limit - pos;
113
+ inflateInit(&stream);
114
+
115
+ for (;;) {
116
+ int ret = inflate(&stream, 0);
117
+ if (ret == Z_STREAM_END)
118
+ break;
119
+ if (ret != Z_OK)
120
+ bad_object(offset, "inflate returned %d", ret);
121
+ }
122
+ inflateEnd(&stream);
123
+ if (stream.total_out != size)
124
+ bad_object(offset, "size mismatch (expected %lu, got %lu)",
125
+ size, stream.total_out);
126
+ *current_pos = pack_limit - stream.avail_in;
127
+ return buf;
128
+}
129
+
130
+static void *unpack_raw_entry(unsigned long offset,
131
+ enum object_type *obj_type,
132
+ unsigned long *obj_size,
133
+ unsigned char *delta_base,
134
+ unsigned long *next_obj_offset)
135
+{
136
+ unsigned long pack_limit = pack_size - 20;
137
+ unsigned long pos = offset;
138
+ unsigned char c;
139
+ unsigned long size;
140
+ unsigned shift;
141
+ enum object_type type;
142
+ void *data;
143
+
144
+ c = pack_base[pos++];
145
+ type = (c >> 4) & 7;
146
+ size = (c & 15);
147
+ shift = 4;
148
+ while (c & 0x80) {
149
+ if (pos >= pack_limit)
150
+ bad_object(offset, "object extends past end of pack");
151
+ c = pack_base[pos++];
152
+ size += (c & 0x7fUL) << shift;
153
+ shift += 7;
154
+ }
155
+
156
+ switch (type) {
157
+ case OBJ_DELTA:
158
+ if (pos + 20 >= pack_limit)
159
+ bad_object(offset, "object extends past end of pack");
160
+ memcpy(delta_base, pack_base + pos, 20);
161
+ pos += 20;
162
+ /* fallthru */
163
+ case OBJ_COMMIT:
164
+ case OBJ_TREE:
165
+ case OBJ_BLOB:
166
+ case OBJ_TAG:
167
+ data = unpack_entry_data(offset, &pos, size);
168
+ break;
169
+ default:
170
+ bad_object(offset, "bad object type %d", type);
171
+ }
172
+
173
+ *obj_type = type;
174
+ *obj_size = size;
175
+ *next_obj_offset = pos;
176
+ return data;
177
+}
178
+
179
+static int find_delta(const unsigned char *base_sha1)
180
+{
181
+ int first = 0, last = nr_deltas;
182
+
183
+ while (first < last) {
184
+ int next = (first + last) / 2;
185
+ struct delta_entry *delta = &deltas[next];
186
+ int cmp;
187
+
188
+ cmp = memcmp(base_sha1, delta->base_sha1, 20);
189
+ if (!cmp)
190
+ return next;
191
+ if (cmp < 0) {
192
+ last = next;
193
+ continue;
194
+ }
195
+ first = next+1;
196
+ }
197
+ return -first-1;
198
+}
199
+
200
+static int find_deltas_based_on_sha1(const unsigned char *base_sha1,
201
+ int *first_index, int *last_index)
202
+{
203
+ int first = find_delta(base_sha1);
204
+ int last = first;
205
+ int end = nr_deltas - 1;
206
+
207
+ if (first < 0)
208
+ return -1;
209
+ while (first > 0 && !memcmp(deltas[first-1].base_sha1, base_sha1, 20))
210
+ --first;
211
+ while (last < end && !memcmp(deltas[last+1].base_sha1, base_sha1, 20))
212
+ ++last;
213
+ *first_index = first;
214
+ *last_index = last;
215
+ return 0;
216
+}
217
+
218
+static void sha1_object(const void *data, unsigned long size,
219
+ enum object_type type, unsigned char *sha1)
220
+{
221
+ SHA_CTX ctx;
222
+ char header[50];
223
+ int header_size;
224
+ const char *type_str;
225
+
226
+ switch (type) {
227
+ case OBJ_COMMIT: type_str = "commit"; break;
228
+ case OBJ_TREE: type_str = "tree"; break;
229
+ case OBJ_BLOB: type_str = "blob"; break;
230
+ case OBJ_TAG: type_str = "tag"; break;
231
+ default:
232
+ die("bad type %d", type);
233
+ }
234
+
235
+ header_size = sprintf(header, "%s %lu", type_str, size) + 1;
236
+
237
+ SHA1_Init(&ctx);
238
+ SHA1_Update(&ctx, header, header_size);
239
+ SHA1_Update(&ctx, data, size);
240
+ SHA1_Final(sha1, &ctx);
241
+}
242
+
243
+static void resolve_delta(struct delta_entry *delta, void *base_data,
244
+ unsigned long base_size, enum object_type type)
245
+{
246
+ struct object_entry *obj = delta->obj;
247
+ void *delta_data;
248
+ unsigned long delta_size;
249
+ void *result;
250
+ unsigned long result_size;
251
+ enum object_type delta_type;
252
+ unsigned char base_sha1[20];
253
+ unsigned long next_obj_offset;
254
+ int j, first, last;
255
+
256
+ obj->real_type = type;
257
+ delta_data = unpack_raw_entry(obj->offset, &delta_type,
258
+ &delta_size, base_sha1,
259
+ &next_obj_offset);
260
+ result = patch_delta(base_data, base_size, delta_data, delta_size,
261
+ &result_size);
262
+ free(delta_data);
263
+ if (!result)
264
+ bad_object(obj->offset, "failed to apply delta");
265
+ sha1_object(result, result_size, type, obj->sha1);
266
+ if (!find_deltas_based_on_sha1(obj->sha1, &first, &last)) {
267
+ for (j = first; j <= last; j++)
268
+ resolve_delta(&deltas[j], result, result_size, type);
269
+ }
270
+ free(result);
271
+}
272
+
273
+static int compare_delta_entry(const void *a, const void *b)
274
+{
275
+ const struct delta_entry *delta_a = a;
276
+ const struct delta_entry *delta_b = b;
277
+ return memcmp(delta_a->base_sha1, delta_b->base_sha1, 20);
278
+}
279
+
280
+static void parse_pack_objects(void)
281
+{
282
+ int i;
283
+ unsigned long offset = sizeof(struct pack_header);
284
+ unsigned char base_sha1[20];
285
+ void *data;
286
+ unsigned long data_size;
287
+
288
+ /*
289
+ * First pass:
290
+ * - find locations of all objects;
291
+ * - calculate SHA1 of all non-delta objects;
292
+ * - remember base SHA1 for all deltas.
293
+ */
294
+ for (i = 0; i < nr_objects; i++) {
295
+ struct object_entry *obj = &objects[i];
296
+ obj->offset = offset;
297
+ data = unpack_raw_entry(offset, &obj->type, &data_size,
298
+ base_sha1, &offset);
299
+ obj->real_type = obj->type;
300
+ if (obj->type == OBJ_DELTA) {
301
+ struct delta_entry *delta = &deltas[nr_deltas++];
302
+ delta->obj = obj;
303
+ memcpy(delta->base_sha1, base_sha1, 20);
304
+ } else
305
+ sha1_object(data, data_size, obj->type, obj->sha1);
306
+ free(data);
307
+ }
308
+ if (offset != pack_size - 20)
309
+ die("packfile '%s' has junk at the end", pack_name);
310
+
311
+ /* Sort deltas by base SHA1 for fast searching */
312
+ qsort(deltas, nr_deltas, sizeof(struct delta_entry),
313
+ compare_delta_entry);
314
+
315
+ /*
316
+ * Second pass:
317
+ * - for all non-delta objects, look if it is used as a base for
318
+ * deltas;
319
+ * - if used as a base, uncompress the object and apply all deltas,
320
+ * recursively checking if the resulting object is used as a base
321
+ * for some more deltas.
322
+ */
323
+ for (i = 0; i < nr_objects; i++) {
324
+ struct object_entry *obj = &objects[i];
325
+ int j, first, last;
326
+
327
+ if (obj->type == OBJ_DELTA)
328
+ continue;
329
+ if (find_deltas_based_on_sha1(obj->sha1, &first, &last))
330
+ continue;
331
+ data = unpack_raw_entry(obj->offset, &obj->type, &data_size,
332
+ base_sha1, &offset);
333
+ for (j = first; j <= last; j++)
334
+ resolve_delta(&deltas[j], data, data_size, obj->type);
335
+ free(data);
336
+ }
337
+
338
+ /* Check for unresolved deltas */
339
+ for (i = 0; i < nr_deltas; i++) {
340
+ if (deltas[i].obj->real_type == OBJ_DELTA)
341
+ die("packfile '%s' has unresolved deltas", pack_name);
342
+ }
343
+}
344
+
345
+static int sha1_compare(const void *_a, const void *_b)
346
+{
347
+ struct object_entry *a = *(struct object_entry **)_a;
348
+ struct object_entry *b = *(struct object_entry **)_b;
349
+ return memcmp(a->sha1, b->sha1, 20);
350
+}
351
+
352
+static void write_index_file(const char *index_name)
353
+{
354
+ struct sha1file *f;
355
+ struct object_entry **sorted_by_sha =
356
+ xcalloc(nr_objects, sizeof(struct object_entry *));
357
+ struct object_entry **list = sorted_by_sha;
358
+ struct object_entry **last = sorted_by_sha + nr_objects;
359
+ unsigned int array[256];
360
+ int i;
361
+
362
+ for (i = 0; i < nr_objects; ++i)
363
+ sorted_by_sha[i] = &objects[i];
364
+ qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
365
+ sha1_compare);
366
+
367
+ unlink(index_name);
368
+ f = sha1create("%s", index_name);
369
+
370
+ /*
371
+ * Write the first-level table (the list is sorted,
372
+ * but we use a 256-entry lookup to be able to avoid
373
+ * having to do eight extra binary search iterations).
374
+ */
375
+ for (i = 0; i < 256; i++) {
376
+ struct object_entry **next = list;
377
+ while (next < last) {
378
+ struct object_entry *obj = *next;
379
+ if (obj->sha1[0] != i)
380
+ break;
381
+ next++;
382
+ }
383
+ array[i] = htonl(next - sorted_by_sha);
384
+ list = next;
385
+ }
386
+ sha1write(f, array, 256 * sizeof(int));
387
+
388
+ /*
389
+ * Write the actual SHA1 entries..
390
+ */
391
+ list = sorted_by_sha;
392
+ for (i = 0; i < nr_objects; i++) {
393
+ struct object_entry *obj = *list++;
394
+ unsigned int offset = htonl(obj->offset);
395
+ sha1write(f, &offset, 4);
396
+ sha1write(f, obj->sha1, 20);
397
+ }
398
+ sha1write(f, pack_base + pack_size - 20, 20);
399
+ sha1close(f, NULL, 1);
400
+ free(sorted_by_sha);
401
+}
402
+
403
+int main(int argc, char **argv)
404
+{
405
+ int i;
406
+ char *index_name = NULL;
407
+ char *index_name_buf = NULL;
408
+
409
+ for (i = 1; i < argc; i++) {
410
+ const char *arg = argv[i];
411
+
412
+ if (*arg == '-') {
413
+ if (!strcmp(arg, "-o")) {
414
+ if (index_name || (i+1) >= argc)
415
+ usage(index_pack_usage);
416
+ index_name = argv[++i];
417
+ } else
418
+ usage(index_pack_usage);
419
+ continue;
420
+ }
421
+
422
+ if (pack_name)
423
+ usage(index_pack_usage);
424
+ pack_name = arg;
425
+ }
426
+
427
+ if (!pack_name)
428
+ usage(index_pack_usage);
429
+ if (!index_name) {
430
+ int len = strlen(pack_name);
431
+ if (len < 5 || strcmp(pack_name + len - 5, ".pack"))
432
+ die("packfile name '%s' does not end with '.pack'",
433
+ pack_name);
434
+ index_name_buf = xmalloc(len - 1);
435
+ memcpy(index_name_buf, pack_name, len - 5);
436
+ strcpy(index_name_buf + len - 5, ".idx");
437
+ index_name = index_name_buf;
438
+ }
439
+
440
+ open_pack_file();
441
+ parse_pack_header();
442
+ objects = xcalloc(nr_objects, sizeof(struct object_entry));
443
+ deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
444
+ parse_pack_objects();
445
+ free(deltas);
446
+ write_index_file(index_name);
447
+ free(objects);
448
+ free(index_name_buf);
449
+
450
+ return 0;
451
+}
t/t5300-pack-object.sh
+18
@@ -165,4 +165,22 @@ test_expect_success \
165
166
:'
167
168
+test_expect_success \
169
+ 'build pack index for an existing pack' \
170
+ 'cp test-1-${packname_1}.pack test-3.pack &&
171
+ git-index-pack -o tmp.idx test-3.pack &&
172
+ cmp tmp.idx test-1-${packname_1}.idx &&
173
+
174
+ git-index-pack test-3.pack &&
175
+ cmp test-3.idx test-1-${packname_1}.idx &&
176
+
177
+ cp test-2-${packname_2}.pack test-3.pack &&
178
+ git-index-pack -o tmp.idx test-2-${packname_2}.pack &&
179
+ cmp tmp.idx test-2-${packname_2}.idx &&
180
+
181
+ git-index-pack test-3.pack &&
182
+ cmp test-3.idx test-2-${packname_2}.idx &&
183
+
184
+ :'
185
+
186
test_done