archive-zip: streaming for deflated files
After an entry has been streamed out, its CRC and sizes are written as part of a data descriptor. For simplicity, we make the buffer for the compressed chunks twice as big as for the uncompressed ones, to be sure the result fit in even if deflate makes them bigger. t5000 verifies output. t1050 makes sure the command always respects core.bigfilethreshold Signed-off-by: Rene Scharfe <rene.scharfe@lsrfire.ath.cx> Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
René Scharfe committed
May 3, 2012 at 08:51 UTC
c743c21591f9433fe784ac38902872701ce2e850
3 files changed
+74
-1
archive-zip.c
+63
-1
@@ -211,7 +211,7 @@ static int write_zip_entry(struct archiver_args *args,
211
compressed_size = size;
212
213
if (S_ISREG(mode) && type == OBJ_BLOB && !args->convert &&
214
- size > big_file_threshold && method == 0) {
214
+ size > big_file_threshold) {
215
stream = open_istream(sha1, &type, &size, NULL);
216
if (!stream)
217
return error("cannot stream blob %s",
@@ -307,6 +307,68 @@ static int write_zip_entry(struct archiver_args *args,
307
write_zip_data_desc(size, compressed_size, crc);
308
zip_offset += ZIP_DATA_DESC_SIZE;
309
310
+ set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
311
+ } else if (stream && method == 8) {
312
+ unsigned char buf[STREAM_BUFFER_SIZE];
313
+ ssize_t readlen;
314
+ git_zstream zstream;
315
+ int result;
316
+ size_t out_len;
317
+ unsigned char compressed[STREAM_BUFFER_SIZE * 2];
318
+
319
+ memset(&zstream, 0, sizeof(zstream));
320
+ git_deflate_init(&zstream, args->compression_level);
321
+
322
+ compressed_size = 0;
323
+ zstream.next_out = compressed;
324
+ zstream.avail_out = sizeof(compressed);
325
+
326
+ for (;;) {
327
+ readlen = read_istream(stream, buf, sizeof(buf));
328
+ if (readlen <= 0)
329
+ break;
330
+ crc = crc32(crc, buf, readlen);
331
+
332
+ zstream.next_in = buf;
333
+ zstream.avail_in = readlen;
334
+ result = git_deflate(&zstream, 0);
335
+ if (result != Z_OK)
336
+ die("deflate error (%d)", result);
337
+ out = compressed;
338
+ if (!compressed_size)
339
+ out += 2;
340
+ out_len = zstream.next_out - out;
341
+
342
+ if (out_len > 0) {
343
+ write_or_die(1, out, out_len);
344
+ compressed_size += out_len;
345
+ zstream.next_out = compressed;
346
+ zstream.avail_out = sizeof(compressed);
347
+ }
348
+
349
+ }
350
+ close_istream(stream);
351
+ if (readlen)
352
+ return readlen;
353
+
354
+ zstream.next_in = buf;
355
+ zstream.avail_in = 0;
356
+ result = git_deflate(&zstream, Z_FINISH);
357
+ if (result != Z_STREAM_END)
358
+ die("deflate error (%d)", result);
359
+
360
+ git_deflate_end(&zstream);
361
+ out = compressed;
362
+ if (!compressed_size)
363
+ out += 2;
364
+ out_len = zstream.next_out - out - 4;
365
+ write_or_die(1, out, out_len);
366
+ compressed_size += out_len;
367
+ zip_offset += compressed_size;
368
+
369
+ write_zip_data_desc(size, compressed_size, crc);
370
+ zip_offset += ZIP_DATA_DESC_SIZE;
371
+
372
set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
373
} else if (compressed_size > 0) {
374
write_or_die(1, out, compressed_size);
t/t1050-large.sh
+4
@@ -142,4 +142,8 @@ test_expect_success 'zip achiving, store only' '
142
git archive --format=zip -0 HEAD >/dev/null
143
'
144
145
+test_expect_success 'zip achiving, deflate' '
146
+ git archive --format=zip HEAD >/dev/null
147
+'
148
+
149
test_done
t/t5000-tar-tree.sh
+7
@@ -250,6 +250,13 @@ test_expect_success UNZIP 'git archive -0 --format=zip on large files' '
250
(mkdir large && cd large && $UNZIP ../large.zip)
251
'
252
253
+test_expect_success UNZIP 'git archive --format=zip on large files' '
254
+ test_config core.bigfilethreshold 1 &&
255
+ git archive --format=zip HEAD >large-compressed.zip &&
256
+ (mkdir large-compressed && cd large-compressed && $UNZIP ../large-compressed.zip) &&
257
+ test_cmp large-compressed/a/bin/sh large/a/bin/sh
258
+'
259
+
260
test_expect_success \
261
'git archive --list outside of a git repo' \
262
'GIT_DIR=some/non-existing/directory git archive --list'