enable index-pack streaming capability
A new flag, --stdin, allows for a pack to be received over a stream. When this flag is provided, the pack content is written to either the named pack file or directly to the object repository under the same name as produced by git-repack. The pack index is written as well with the corresponding base name, unless the index name is overriden with -o. With this patch, git-index-pack could be used instead of git-unpack-objects when fetching remote objects but only with non "thin" packs for now. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nicolas Pitre committed
Oct 23, 2006 at 14:50 UTC
e42797f5b6d5ed3b8c894d89493e285c40d58dc8
2 files changed
+111
-30
Documentation/git-index-pack.txt
+7
-1
@@ -8,7 +8,7 @@ git-index-pack - Build pack index file for an existing packed archive
8
9
SYNOPSIS
10
--------
11
-'git-index-pack' [-o <index-file>] <pack-file>
11
+'git-index-pack' [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }
12
13
14
DESCRIPTION
@@ -29,6 +29,12 @@ OPTIONS
29
fails if the name of packed archive does not end
30
with .pack).
31
32
+--stdin::
33
+ When this flag is provided, the pack is read from stdin
34
+ instead and a copy is then written to <pack-file>. If
35
+ <pack-file> is not specified, the pack is written to
36
+ objects/pack/ directory of the current git repository with
37
+ a default name determined from the pack content.
38
39
Author
40
------
index-pack.c
+104
-29
@@ -8,7 +8,7 @@
8
#include "tree.h"
9
10
static const char index_pack_usage[] =
11
-"git-index-pack [-o index-file] pack-file";
11
+"git-index-pack [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }";
12
13
struct object_entry
14
{
@@ -37,17 +37,18 @@ struct delta_entry
37
union delta_base base;
38
};
39
40
-static const char *pack_name;
40
static struct object_entry *objects;
41
static struct delta_entry *deltas;
42
static int nr_objects;
43
static int nr_deltas;
44
45
+static int from_stdin;
46
+
47
/* We always read in 4kB chunks. */
48
static unsigned char input_buffer[4096];
49
static unsigned long input_offset, input_len, consumed_bytes;
50
static SHA_CTX input_ctx;
50
-static int input_fd;
51
+static int input_fd, output_fd, mmap_fd;
52
53
/*
54
* Make sure at least "min" bytes are available in the buffer, and
@@ -60,6 +61,8 @@ static void * fill(int min)
61
if (min > sizeof(input_buffer))
62
die("cannot fill %d bytes", min);
63
if (input_offset) {
64
+ if (output_fd >= 0)
65
+ write_or_die(output_fd, input_buffer, input_offset);
66
SHA1_Update(&input_ctx, input_buffer, input_offset);
67
memcpy(input_buffer, input_buffer + input_offset, input_len);
68
input_offset = 0;
@@ -86,13 +89,31 @@ static void use(int bytes)
89
consumed_bytes += bytes;
90
}
91
89
-static void open_pack_file(void)
92
+static const char * open_pack_file(const char *pack_name)
93
{
91
- input_fd = open(pack_name, O_RDONLY);
92
- if (input_fd < 0)
93
- die("cannot open packfile '%s': %s", pack_name,
94
- strerror(errno));
94
+ if (from_stdin) {
95
+ input_fd = 0;
96
+ if (!pack_name) {
97
+ static char tmpfile[PATH_MAX];
98
+ snprintf(tmpfile, sizeof(tmpfile),
99
+ "%s/pack_XXXXXX", get_object_directory());
100
+ output_fd = mkstemp(tmpfile);
101
+ pack_name = xstrdup(tmpfile);
102
+ } else
103
+ output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
104
+ if (output_fd < 0)
105
+ die("unable to create %s: %s\n", pack_name, strerror(errno));
106
+ mmap_fd = output_fd;
107
+ } else {
108
+ input_fd = open(pack_name, O_RDONLY);
109
+ if (input_fd < 0)
110
+ die("cannot open packfile '%s': %s",
111
+ pack_name, strerror(errno));
112
+ output_fd = -1;
113
+ mmap_fd = input_fd;
114
+ }
115
SHA1_Init(&input_ctx);
116
+ return pack_name;
117
}
118
119
static void parse_pack_header(void)
@@ -101,10 +122,9 @@ static void parse_pack_header(void)
122
123
/* Header consistency check */
124
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
104
- die("packfile '%s' signature mismatch", pack_name);
125
+ die("pack signature mismatch");
126
if (!pack_version_ok(hdr->hdr_version))
106
- die("packfile '%s' version %d unsupported",
107
- pack_name, ntohl(hdr->hdr_version));
127
+ die("pack version %d unsupported", ntohl(hdr->hdr_version));
128
129
nr_objects = ntohl(hdr->hdr_entries);
130
use(sizeof(struct pack_header));
@@ -122,8 +142,7 @@ static void bad_object(unsigned long offset, const char *format, ...)
142
va_start(params, format);
143
vsnprintf(buf, sizeof(buf), format, params);
144
va_end(params);
125
- die("packfile '%s': bad object at offset %lu: %s",
126
- pack_name, offset, buf);
145
+ die("pack has bad object at offset %lu: %s", offset, buf);
146
}
147
148
static void *unpack_entry_data(unsigned long offset, unsigned long size)
@@ -222,9 +241,9 @@ static void * get_data_from_pack(struct object_entry *obj)
241
int st;
242
243
map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,
225
- input_fd, from - pg_offset);
244
+ mmap_fd, from - pg_offset);
245
if (map == MAP_FAILED)
227
- die("cannot mmap packfile '%s': %s", pack_name, strerror(errno));
246
+ die("cannot mmap pack file: %s", strerror(errno));
247
data = xmalloc(obj->size);
248
memset(&stream, 0, sizeof(stream));
249
stream.next_out = data;
@@ -382,14 +401,16 @@ static void parse_pack_objects(unsigned char *sha1)
401
SHA1_Update(&input_ctx, input_buffer, input_offset);
402
SHA1_Final(sha1, &input_ctx);
403
if (hashcmp(fill(20), sha1))
385
- die("packfile '%s' SHA1 mismatch", pack_name);
404
+ die("pack is corrupted (SHA1 mismatch)");
405
use(20);
406
+ if (output_fd >= 0)
407
+ write_or_die(output_fd, input_buffer, input_offset);
408
409
/* If input_fd is a file, we should have reached its end now. */
410
if (fstat(input_fd, &st))
390
- die("cannot fstat packfile '%s': %s", pack_name, strerror(errno));
411
+ die("cannot fstat packfile: %s", strerror(errno));
412
if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)
392
- die("packfile '%s' has junk at the end", pack_name);
413
+ die("pack has junk at the end");
414
415
/* Sort deltas by base SHA1/offset for fast searching */
416
qsort(deltas, nr_deltas, sizeof(struct delta_entry),
@@ -435,7 +456,7 @@ static void parse_pack_objects(unsigned char *sha1)
456
for (i = 0; i < nr_deltas; i++) {
457
if (deltas[i].obj->real_type == OBJ_REF_DELTA ||
458
deltas[i].obj->real_type == OBJ_OFS_DELTA)
438
- die("packfile '%s' has unresolved deltas", pack_name);
459
+ die("pack has unresolved deltas");
460
}
461
}
462
@@ -450,12 +471,12 @@ static int sha1_compare(const void *_a, const void *_b)
471
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
472
* the SHA1 hash of sorted object names.
473
*/
453
-static void write_index_file(const char *index_name, unsigned char *sha1)
474
+static const char * write_index_file(const char *index_name, unsigned char *sha1)
475
{
476
struct sha1file *f;
477
struct object_entry **sorted_by_sha, **list, **last;
478
unsigned int array[256];
458
- int i;
479
+ int i, fd;
480
SHA_CTX ctx;
481
482
if (nr_objects) {
@@ -472,8 +493,19 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
493
else
494
sorted_by_sha = list = last = NULL;
495
475
- unlink(index_name);
476
- f = sha1create("%s", index_name);
496
+ if (!index_name) {
497
+ static char tmpfile[PATH_MAX];
498
+ snprintf(tmpfile, sizeof(tmpfile),
499
+ "%s/index_XXXXXX", get_object_directory());
500
+ fd = mkstemp(tmpfile);
501
+ index_name = xstrdup(tmpfile);
502
+ } else {
503
+ unlink(index_name);
504
+ fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
505
+ }
506
+ if (fd < 0)
507
+ die("unable to create %s: %s", index_name, strerror(errno));
508
+ f = sha1fd(fd, index_name);
509
510
/*
511
* Write the first-level table (the list is sorted,
@@ -513,12 +545,52 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
545
sha1close(f, NULL, 1);
546
free(sorted_by_sha);
547
SHA1_Final(sha1, &ctx);
548
+ return index_name;
549
+}
550
+
551
+static void final(const char *final_pack_name, const char *curr_pack_name,
552
+ const char *final_index_name, const char *curr_index_name,
553
+ unsigned char *sha1)
554
+{
555
+ char name[PATH_MAX];
556
+ int err;
557
+
558
+ if (!from_stdin) {
559
+ close(input_fd);
560
+ } else {
561
+ err = close(output_fd);
562
+ if (err)
563
+ die("error while closing pack file: %s", strerror(errno));
564
+ chmod(curr_pack_name, 0444);
565
+ }
566
+
567
+ if (final_pack_name != curr_pack_name) {
568
+ if (!final_pack_name) {
569
+ snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
570
+ get_object_directory(), sha1_to_hex(sha1));
571
+ final_pack_name = name;
572
+ }
573
+ if (move_temp_to_file(curr_pack_name, final_pack_name))
574
+ die("cannot store pack file");
575
+ }
576
+
577
+ chmod(curr_index_name, 0444);
578
+ if (final_index_name != curr_index_name) {
579
+ if (!final_index_name) {
580
+ snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
581
+ get_object_directory(), sha1_to_hex(sha1));
582
+ final_index_name = name;
583
+ }
584
+ if (move_temp_to_file(curr_index_name, final_index_name))
585
+ die("cannot store index file");
586
+ }
587
}
588
589
int main(int argc, char **argv)
590
{
591
int i;
521
- char *index_name = NULL;
592
+ const char *curr_pack, *pack_name = NULL;
593
+ const char *curr_index, *index_name = NULL;
594
char *index_name_buf = NULL;
595
unsigned char sha1[20];
596
@@ -526,7 +598,9 @@ int main(int argc, char **argv)
598
const char *arg = argv[i];
599
600
if (*arg == '-') {
529
- if (!strcmp(arg, "-o")) {
601
+ if (!strcmp(arg, "--stdin")) {
602
+ from_stdin = 1;
603
+ } else if (!strcmp(arg, "-o")) {
604
if (index_name || (i+1) >= argc)
605
usage(index_pack_usage);
606
index_name = argv[++i];
@@ -540,9 +614,9 @@ int main(int argc, char **argv)
614
pack_name = arg;
615
}
616
543
- if (!pack_name)
617
+ if (!pack_name && !from_stdin)
618
usage(index_pack_usage);
545
- if (!index_name) {
619
+ if (!index_name && pack_name) {
620
int len = strlen(pack_name);
621
if (!has_extension(pack_name, ".pack"))
622
die("packfile name '%s' does not end with '.pack'",
@@ -553,13 +627,14 @@ int main(int argc, char **argv)
627
index_name = index_name_buf;
628
}
629
556
- open_pack_file();
630
+ curr_pack = open_pack_file(pack_name);
631
parse_pack_header();
632
objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
633
deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
634
parse_pack_objects(sha1);
635
free(deltas);
562
- write_index_file(index_name, sha1);
636
+ curr_index = write_index_file(index_name, sha1);
637
+ final(pack_name, curr_pack, index_name, curr_index, sha1);
638
free(objects);
639
free(index_name_buf);
640