archive-tar: stream large blobs to tar file
t5000 verifies output while t1050 makes sure the command always respects core.bigfilethreshold Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
May 3, 2012 at 08:51 UTC
5544049def9a80bc5ea09a5649e13c1b56160518
3 files changed
+61
-5
archive-tar.c
+51
-5
@@ -4,6 +4,7 @@
4
#include "cache.h"
5
#include "tar.h"
6
#include "archive.h"
7
+#include "streaming.h"
8
#include "run-command.h"
9
10
#define RECORDSIZE (512)
@@ -30,10 +31,9 @@ static void write_if_needed(void)
31
* queues up writes, so that all our write(2) calls write exactly one
32
* full block; pads writes to RECORDSIZE
33
*/
33
-static void write_blocked(const void *data, unsigned long size)
34
+static void do_write_blocked(const void *data, unsigned long size)
35
{
36
const char *buf = data;
36
- unsigned long tail;
37
38
if (offset) {
39
unsigned long chunk = BLOCKSIZE - offset;
@@ -54,6 +54,11 @@ static void write_blocked(const void *data, unsigned long size)
54
memcpy(block + offset, buf, size);
55
offset += size;
56
}
57
+}
58
+
59
+static void finish_record(void)
60
+{
61
+ unsigned long tail;
62
tail = offset % RECORDSIZE;
63
if (tail) {
64
memset(block + offset, 0, RECORDSIZE - tail);
@@ -62,6 +67,12 @@ static void write_blocked(const void *data, unsigned long size)
67
write_if_needed();
68
}
69
70
+static void write_blocked(const void *data, unsigned long size)
71
+{
72
+ do_write_blocked(data, size);
73
+ finish_record();
74
+}
75
+
76
/*
77
* The end of tar archives is marked by 2*512 nul bytes and after that
78
* follows the rest of the block (if any).
@@ -77,6 +88,33 @@ static void write_trailer(void)
88
}
89
}
90
91
+/*
92
+ * queues up writes, so that all our write(2) calls write exactly one
93
+ * full block; pads writes to RECORDSIZE
94
+ */
95
+static int stream_blocked(const unsigned char *sha1)
96
+{
97
+ struct git_istream *st;
98
+ enum object_type type;
99
+ unsigned long sz;
100
+ char buf[BLOCKSIZE];
101
+ ssize_t readlen;
102
+
103
+ st = open_istream(sha1, &type, &sz, NULL);
104
+ if (!st)
105
+ return error("cannot stream blob %s", sha1_to_hex(sha1));
106
+ for (;;) {
107
+ readlen = read_istream(st, buf, sizeof(buf));
108
+ if (readlen <= 0)
109
+ break;
110
+ do_write_blocked(buf, readlen);
111
+ }
112
+ close_istream(st);
113
+ if (!readlen)
114
+ finish_record();
115
+ return readlen;
116
+}
117
+
118
/*
119
* pax extended header records have the format "%u %s=%s\n". %u contains
120
* the size of the whole string (including the %u), the first %s is the
@@ -203,7 +241,11 @@ static int write_tar_entry(struct archiver_args *args,
241
} else
242
memcpy(header.name, path, pathlen);
243
206
- if (S_ISLNK(mode) || S_ISREG(mode)) {
244
+ if (S_ISREG(mode) && !args->convert &&
245
+ sha1_object_info(sha1, &size) == OBJ_BLOB &&
246
+ size > big_file_threshold)
247
+ buffer = NULL;
248
+ else if (S_ISLNK(mode) || S_ISREG(mode)) {
249
enum object_type type;
250
buffer = sha1_file_to_archive(args, path, sha1, old_mode, &type, &size);
251
if (!buffer)
@@ -235,8 +277,12 @@ static int write_tar_entry(struct archiver_args *args,
277
}
278
strbuf_release(&ext_header);
279
write_blocked(&header, sizeof(header));
238
- if (S_ISREG(mode) && buffer && size > 0)
239
- write_blocked(buffer, size);
280
+ if (S_ISREG(mode) && size > 0) {
281
+ if (buffer)
282
+ write_blocked(buffer, size);
283
+ else
284
+ err = stream_blocked(sha1);
285
+ }
286
free(buffer);
287
return err;
288
}
t/t1050-large.sh
+4
@@ -134,4 +134,8 @@ test_expect_success 'repack' '
134
git repack -ad
135
'
136
137
+test_expect_success 'tar achiving' '
138
+ git archive --format=tar HEAD >/dev/null
139
+'
140
+
141
test_done
t/t5000-tar-tree.sh
+6
@@ -84,6 +84,12 @@ test_expect_success \
84
'git archive vs. git tar-tree' \
85
'test_cmp b.tar b2.tar'
86
87
+test_expect_success 'git archive on large files' '
88
+ test_config core.bigfilethreshold 1 &&
89
+ git archive HEAD >b3.tar &&
90
+ test_cmp b.tar b3.tar
91
+'
92
+
93
test_expect_success \
94
'git archive in a bare repo' \
95
'(cd bare.git && git archive HEAD) >b3.tar'