pack-objects: use streaming interface for reading large loose blobs
git usually streams large blobs directly to packs. But there are cases where git can create large loose blobs (unpack-objects or hash-object over pipe). Or they can come from other git implementations. core.bigfilethreshold can also be lowered down and introduce a new wave of large loose blobs. Use streaming interface to read/compress/write these blobs in one go. Fall back to normal way if somehow streaming interface cannot be used. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
May 26, 2012 at 17:28 UTC
cf2ba13ac6a7589fde5b7e888fedb2e7299e423a
2 files changed
+79
-6
builtin/pack-objects.c
+67
-6
@@ -16,6 +16,7 @@
16
#include "list-objects.h"
17
#include "progress.h"
18
#include "refs.h"
19
+#include "streaming.h"
20
#include "thread-utils.h"
21
22
static const char *pack_usage[] = {
@@ -150,6 +151,46 @@ static unsigned long do_compress(void **pptr, unsigned long size)
151
return stream.total_out;
152
}
153
154
+static unsigned long write_large_blob_data(struct git_istream *st, struct sha1file *f,
155
+ const unsigned char *sha1)
156
+{
157
+ git_zstream stream;
158
+ unsigned char ibuf[1024 * 16];
159
+ unsigned char obuf[1024 * 16];
160
+ unsigned long olen = 0;
161
+
162
+ memset(&stream, 0, sizeof(stream));
163
+ git_deflate_init(&stream, pack_compression_level);
164
+
165
+ for (;;) {
166
+ ssize_t readlen;
167
+ int zret = Z_OK;
168
+ readlen = read_istream(st, ibuf, sizeof(ibuf));
169
+ if (readlen == -1)
170
+ die(_("unable to read %s"), sha1_to_hex(sha1));
171
+
172
+ stream.next_in = ibuf;
173
+ stream.avail_in = readlen;
174
+ while ((stream.avail_in || readlen == 0) &&
175
+ (zret == Z_OK || zret == Z_BUF_ERROR)) {
176
+ stream.next_out = obuf;
177
+ stream.avail_out = sizeof(obuf);
178
+ zret = git_deflate(&stream, readlen ? 0 : Z_FINISH);
179
+ sha1write(f, obuf, stream.next_out - obuf);
180
+ olen += stream.next_out - obuf;
181
+ }
182
+ if (stream.avail_in)
183
+ die(_("deflate error (%d)"), zret);
184
+ if (readlen == 0) {
185
+ if (zret != Z_STREAM_END)
186
+ die(_("deflate error (%d)"), zret);
187
+ break;
188
+ }
189
+ }
190
+ git_deflate_end(&stream);
191
+ return olen;
192
+}
193
+
194
/*
195
* we are going to reuse the existing object data as is. make
196
* sure it is not corrupt.
@@ -208,11 +249,18 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent
249
unsigned hdrlen;
250
enum object_type type;
251
void *buf;
252
+ struct git_istream *st = NULL;
253
254
if (!usable_delta) {
213
- buf = read_sha1_file(entry->idx.sha1, &type, &size);
214
- if (!buf)
215
- die("unable to read %s", sha1_to_hex(entry->idx.sha1));
255
+ if (entry->type == OBJ_BLOB &&
256
+ entry->size > big_file_threshold &&
257
+ (st = open_istream(entry->idx.sha1, &type, &size, NULL)) != NULL)
258
+ buf = NULL;
259
+ else {
260
+ buf = read_sha1_file(entry->idx.sha1, &type, &size);
261
+ if (!buf)
262
+ die(_("unable to read %s"), sha1_to_hex(entry->idx.sha1));
263
+ }
264
/*
265
* make sure no cached delta data remains from a
266
* previous attempt before a pack split occurred.
@@ -233,7 +281,9 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent
281
OBJ_OFS_DELTA : OBJ_REF_DELTA;
282
}
283
236
- if (entry->z_delta_size)
284
+ if (st) /* large blob case, just assume we don't compress well */
285
+ datalen = size;
286
+ else if (entry->z_delta_size)
287
datalen = entry->z_delta_size;
288
else
289
datalen = do_compress(&buf, size);
@@ -256,6 +306,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent
306
while (ofs >>= 7)
307
dheader[--pos] = 128 | (--ofs & 127);
308
if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
309
+ if (st)
310
+ close_istream(st);
311
free(buf);
312
return 0;
313
}
@@ -268,6 +320,8 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent
320
* an additional 20 bytes for the base sha1.
321
*/
322
if (limit && hdrlen + 20 + datalen + 20 >= limit) {
323
+ if (st)
324
+ close_istream(st);
325
free(buf);
326
return 0;
327
}
@@ -276,13 +330,20 @@ static unsigned long write_no_reuse_object(struct sha1file *f, struct object_ent
330
hdrlen += 20;
331
} else {
332
if (limit && hdrlen + datalen + 20 >= limit) {
333
+ if (st)
334
+ close_istream(st);
335
free(buf);
336
return 0;
337
}
338
sha1write(f, header, hdrlen);
339
}
284
- sha1write(f, buf, datalen);
285
- free(buf);
340
+ if (st) {
341
+ datalen = write_large_blob_data(st, f, entry->idx.sha1);
342
+ close_istream(st);
343
+ } else {
344
+ sha1write(f, buf, datalen);
345
+ free(buf);
346
+ }
347
348
return hdrlen + datalen;
349
}
t/t1050-large.sh
+12
@@ -134,6 +134,18 @@ test_expect_success 'repack' '
134
git repack -ad
135
'
136
137
+test_expect_success 'pack-objects with large loose object' '
138
+ SHA1=`git hash-object huge` &&
139
+ test_create_repo loose &&
140
+ echo $SHA1 | git pack-objects --stdout |
141
+ GIT_ALLOC_LIMIT=0 GIT_DIR=loose/.git git unpack-objects &&
142
+ echo $SHA1 | GIT_DIR=loose/.git git pack-objects pack &&
143
+ test_create_repo packed &&
144
+ mv pack-* packed/.git/objects/pack &&
145
+ GIT_DIR=packed/.git git cat-file blob $SHA1 >actual &&
146
+ cmp huge actual
147
+'
148
+
149
test_expect_success 'tar achiving' '
150
git archive --format=tar HEAD >/dev/null
151
'