@matheusbsbs / matheus-bagliano / commits / e42797f5b6
Fork 0

enable index-pack streaming capability

A new flag, --stdin, allows for a pack to be received over a stream. When this flag is provided, the pack content is written to either the named pack file or directly to the object repository under the same name as produced by git-repack. The pack index is written as well with the corresponding base name, unless the index name is overriden with -o. With this patch, git-index-pack could be used instead of git-unpack-objects when fetching remote objects but only with non "thin" packs for now. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Oct 23, 2006 at 14:50 UTC e42797f5b6d5ed3b8c894d89493e285c40d58dc8
2 files changed +111 -30
Documentation/git-index-pack.txt
+7 -1
@@ -8,7 +8,7 @@ git-index-pack - Build pack index file for an existing packed archive
8
9 SYNOPSIS
10 --------
11 -'git-index-pack' [-o <index-file>] <pack-file>
11 +'git-index-pack' [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }
12
13
14 DESCRIPTION
@@ -29,6 +29,12 @@ OPTIONS
29 fails if the name of packed archive does not end
30 with .pack).
31
32 +--stdin::
33 + When this flag is provided, the pack is read from stdin
34 + instead and a copy is then written to <pack-file>. If
35 + <pack-file> is not specified, the pack is written to
36 + objects/pack/ directory of the current git repository with
37 + a default name determined from the pack content.
38
39 Author
40 ------
index-pack.c
+104 -29
@@ -8,7 +8,7 @@
8 #include "tree.h"
9
10 static const char index_pack_usage[] =
11 -"git-index-pack [-o index-file] pack-file";
11 +"git-index-pack [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }";
12
13 struct object_entry
14 {
@@ -37,17 +37,18 @@ struct delta_entry
37 union delta_base base;
38 };
39
40 -static const char *pack_name;
40 static struct object_entry *objects;
41 static struct delta_entry *deltas;
42 static int nr_objects;
43 static int nr_deltas;
44
45 +static int from_stdin;
46 +
47 /* We always read in 4kB chunks. */
48 static unsigned char input_buffer[4096];
49 static unsigned long input_offset, input_len, consumed_bytes;
50 static SHA_CTX input_ctx;
50 -static int input_fd;
51 +static int input_fd, output_fd, mmap_fd;
52
53 /*
54 * Make sure at least "min" bytes are available in the buffer, and
@@ -60,6 +61,8 @@ static void * fill(int min)
61 if (min > sizeof(input_buffer))
62 die("cannot fill %d bytes", min);
63 if (input_offset) {
64 + if (output_fd >= 0)
65 + write_or_die(output_fd, input_buffer, input_offset);
66 SHA1_Update(&input_ctx, input_buffer, input_offset);
67 memcpy(input_buffer, input_buffer + input_offset, input_len);
68 input_offset = 0;
@@ -86,13 +89,31 @@ static void use(int bytes)
89 consumed_bytes += bytes;
90 }
91
89 -static void open_pack_file(void)
92 +static const char * open_pack_file(const char *pack_name)
93 {
91 - input_fd = open(pack_name, O_RDONLY);
92 - if (input_fd < 0)
93 - die("cannot open packfile '%s': %s", pack_name,
94 - strerror(errno));
94 + if (from_stdin) {
95 + input_fd = 0;
96 + if (!pack_name) {
97 + static char tmpfile[PATH_MAX];
98 + snprintf(tmpfile, sizeof(tmpfile),
99 + "%s/pack_XXXXXX", get_object_directory());
100 + output_fd = mkstemp(tmpfile);
101 + pack_name = xstrdup(tmpfile);
102 + } else
103 + output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
104 + if (output_fd < 0)
105 + die("unable to create %s: %s\n", pack_name, strerror(errno));
106 + mmap_fd = output_fd;
107 + } else {
108 + input_fd = open(pack_name, O_RDONLY);
109 + if (input_fd < 0)
110 + die("cannot open packfile '%s': %s",
111 + pack_name, strerror(errno));
112 + output_fd = -1;
113 + mmap_fd = input_fd;
114 + }
115 SHA1_Init(&input_ctx);
116 + return pack_name;
117 }
118
119 static void parse_pack_header(void)
@@ -101,10 +122,9 @@ static void parse_pack_header(void)
122
123 /* Header consistency check */
124 if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
104 - die("packfile '%s' signature mismatch", pack_name);
125 + die("pack signature mismatch");
126 if (!pack_version_ok(hdr->hdr_version))
106 - die("packfile '%s' version %d unsupported",
107 - pack_name, ntohl(hdr->hdr_version));
127 + die("pack version %d unsupported", ntohl(hdr->hdr_version));
128
129 nr_objects = ntohl(hdr->hdr_entries);
130 use(sizeof(struct pack_header));
@@ -122,8 +142,7 @@ static void bad_object(unsigned long offset, const char *format, ...)
142 va_start(params, format);
143 vsnprintf(buf, sizeof(buf), format, params);
144 va_end(params);
125 - die("packfile '%s': bad object at offset %lu: %s",
126 - pack_name, offset, buf);
145 + die("pack has bad object at offset %lu: %s", offset, buf);
146 }
147
148 static void *unpack_entry_data(unsigned long offset, unsigned long size)
@@ -222,9 +241,9 @@ static void * get_data_from_pack(struct object_entry *obj)
241 int st;
242
243 map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,
225 - input_fd, from - pg_offset);
244 + mmap_fd, from - pg_offset);
245 if (map == MAP_FAILED)
227 - die("cannot mmap packfile '%s': %s", pack_name, strerror(errno));
246 + die("cannot mmap pack file: %s", strerror(errno));
247 data = xmalloc(obj->size);
248 memset(&stream, 0, sizeof(stream));
249 stream.next_out = data;
@@ -382,14 +401,16 @@ static void parse_pack_objects(unsigned char *sha1)
401 SHA1_Update(&input_ctx, input_buffer, input_offset);
402 SHA1_Final(sha1, &input_ctx);
403 if (hashcmp(fill(20), sha1))
385 - die("packfile '%s' SHA1 mismatch", pack_name);
404 + die("pack is corrupted (SHA1 mismatch)");
405 use(20);
406 + if (output_fd >= 0)
407 + write_or_die(output_fd, input_buffer, input_offset);
408
409 /* If input_fd is a file, we should have reached its end now. */
410 if (fstat(input_fd, &st))
390 - die("cannot fstat packfile '%s': %s", pack_name, strerror(errno));
411 + die("cannot fstat packfile: %s", strerror(errno));
412 if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)
392 - die("packfile '%s' has junk at the end", pack_name);
413 + die("pack has junk at the end");
414
415 /* Sort deltas by base SHA1/offset for fast searching */
416 qsort(deltas, nr_deltas, sizeof(struct delta_entry),
@@ -435,7 +456,7 @@ static void parse_pack_objects(unsigned char *sha1)
456 for (i = 0; i < nr_deltas; i++) {
457 if (deltas[i].obj->real_type == OBJ_REF_DELTA ||
458 deltas[i].obj->real_type == OBJ_OFS_DELTA)
438 - die("packfile '%s' has unresolved deltas", pack_name);
459 + die("pack has unresolved deltas");
460 }
461 }
462
@@ -450,12 +471,12 @@ static int sha1_compare(const void *_a, const void *_b)
471 * On entry *sha1 contains the pack content SHA1 hash, on exit it is
472 * the SHA1 hash of sorted object names.
473 */
453 -static void write_index_file(const char *index_name, unsigned char *sha1)
474 +static const char * write_index_file(const char *index_name, unsigned char *sha1)
475 {
476 struct sha1file *f;
477 struct object_entry **sorted_by_sha, **list, **last;
478 unsigned int array[256];
458 - int i;
479 + int i, fd;
480 SHA_CTX ctx;
481
482 if (nr_objects) {
@@ -472,8 +493,19 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
493 else
494 sorted_by_sha = list = last = NULL;
495
475 - unlink(index_name);
476 - f = sha1create("%s", index_name);
496 + if (!index_name) {
497 + static char tmpfile[PATH_MAX];
498 + snprintf(tmpfile, sizeof(tmpfile),
499 + "%s/index_XXXXXX", get_object_directory());
500 + fd = mkstemp(tmpfile);
501 + index_name = xstrdup(tmpfile);
502 + } else {
503 + unlink(index_name);
504 + fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
505 + }
506 + if (fd < 0)
507 + die("unable to create %s: %s", index_name, strerror(errno));
508 + f = sha1fd(fd, index_name);
509
510 /*
511 * Write the first-level table (the list is sorted,
@@ -513,12 +545,52 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
545 sha1close(f, NULL, 1);
546 free(sorted_by_sha);
547 SHA1_Final(sha1, &ctx);
548 + return index_name;
549 +}
550 +
551 +static void final(const char *final_pack_name, const char *curr_pack_name,
552 + const char *final_index_name, const char *curr_index_name,
553 + unsigned char *sha1)
554 +{
555 + char name[PATH_MAX];
556 + int err;
557 +
558 + if (!from_stdin) {
559 + close(input_fd);
560 + } else {
561 + err = close(output_fd);
562 + if (err)
563 + die("error while closing pack file: %s", strerror(errno));
564 + chmod(curr_pack_name, 0444);
565 + }
566 +
567 + if (final_pack_name != curr_pack_name) {
568 + if (!final_pack_name) {
569 + snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
570 + get_object_directory(), sha1_to_hex(sha1));
571 + final_pack_name = name;
572 + }
573 + if (move_temp_to_file(curr_pack_name, final_pack_name))
574 + die("cannot store pack file");
575 + }
576 +
577 + chmod(curr_index_name, 0444);
578 + if (final_index_name != curr_index_name) {
579 + if (!final_index_name) {
580 + snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
581 + get_object_directory(), sha1_to_hex(sha1));
582 + final_index_name = name;
583 + }
584 + if (move_temp_to_file(curr_index_name, final_index_name))
585 + die("cannot store index file");
586 + }
587 }
588
589 int main(int argc, char **argv)
590 {
591 int i;
521 - char *index_name = NULL;
592 + const char *curr_pack, *pack_name = NULL;
593 + const char *curr_index, *index_name = NULL;
594 char *index_name_buf = NULL;
595 unsigned char sha1[20];
596
@@ -526,7 +598,9 @@ int main(int argc, char **argv)
598 const char *arg = argv[i];
599
600 if (*arg == '-') {
529 - if (!strcmp(arg, "-o")) {
601 + if (!strcmp(arg, "--stdin")) {
602 + from_stdin = 1;
603 + } else if (!strcmp(arg, "-o")) {
604 if (index_name || (i+1) >= argc)
605 usage(index_pack_usage);
606 index_name = argv[++i];
@@ -540,9 +614,9 @@ int main(int argc, char **argv)
614 pack_name = arg;
615 }
616
543 - if (!pack_name)
617 + if (!pack_name && !from_stdin)
618 usage(index_pack_usage);
545 - if (!index_name) {
619 + if (!index_name && pack_name) {
620 int len = strlen(pack_name);
621 if (!has_extension(pack_name, ".pack"))
622 die("packfile name '%s' does not end with '.pack'",
@@ -553,13 +627,14 @@ int main(int argc, char **argv)
627 index_name = index_name_buf;
628 }
629
556 - open_pack_file();
630 + curr_pack = open_pack_file(pack_name);
631 parse_pack_header();
632 objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
633 deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
634 parse_pack_objects(sha1);
635 free(deltas);
562 - write_index_file(index_name, sha1);
636 + curr_index = write_index_file(index_name, sha1);
637 + final(pack_name, curr_pack, index_name, curr_index, sha1);
638 free(objects);
639 free(index_name_buf);
640