archive-zip: support files bigger than 4GB
Write a zip64 extended information extra field for big files as part of their local headers and as part of their central directory headers. Also write a zip64 version of the data descriptor in that case. If we're streaming then we don't know the compressed size at the time we write the header. Deflate can end up making a file bigger instead of smaller if we're unlucky. Write a local zip64 header already for files with a size of 2GB or more in this case to be on the safe side. Both sizes need to be included in the local zip64 header, but the extra field for the directory must only contain 64-bit equivalents for 32-bit values of 0xffffffff. Signed-off-by: Rene Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
René Scharfe committed
Apr 24, 2017 at 19:33 UTC
4cdf3f9d84568da72f1dcade812de7a42ecb6d15
2 files changed
+76
-16
archive-zip.c
+75
-15
@@ -45,6 +45,14 @@ struct zip_data_desc {
45
unsigned char _end[1];
46
};
47
48
+struct zip64_data_desc {
49
+ unsigned char magic[4];
50
+ unsigned char crc32[4];
51
+ unsigned char compressed_size[8];
52
+ unsigned char size[8];
53
+ unsigned char _end[1];
54
+};
55
+
56
struct zip_dir_trailer {
57
unsigned char magic[4];
58
unsigned char disk[2];
@@ -65,6 +73,14 @@ struct zip_extra_mtime {
73
unsigned char _end[1];
74
};
75
76
+struct zip64_extra {
77
+ unsigned char magic[2];
78
+ unsigned char extra_size[2];
79
+ unsigned char size[8];
80
+ unsigned char compressed_size[8];
81
+ unsigned char _end[1];
82
+};
83
+
84
struct zip64_dir_trailer {
85
unsigned char magic[4];
86
unsigned char record_size[8];
@@ -94,11 +110,15 @@ struct zip64_dir_trailer_locator {
110
*/
111
#define ZIP_LOCAL_HEADER_SIZE offsetof(struct zip_local_header, _end)
112
#define ZIP_DATA_DESC_SIZE offsetof(struct zip_data_desc, _end)
113
+#define ZIP64_DATA_DESC_SIZE offsetof(struct zip64_data_desc, _end)
114
#define ZIP_DIR_HEADER_SIZE offsetof(struct zip_dir_header, _end)
115
#define ZIP_DIR_TRAILER_SIZE offsetof(struct zip_dir_trailer, _end)
116
#define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end)
117
#define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
118
(ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
119
+#define ZIP64_EXTRA_SIZE offsetof(struct zip64_extra, _end)
120
+#define ZIP64_EXTRA_PAYLOAD_SIZE \
121
+ (ZIP64_EXTRA_SIZE - offsetof(struct zip64_extra, size))
122
#define ZIP64_DIR_TRAILER_SIZE offsetof(struct zip64_dir_trailer, _end)
123
#define ZIP64_DIR_TRAILER_RECORD_SIZE \
124
(ZIP64_DIR_TRAILER_SIZE - \
@@ -202,13 +222,23 @@ static void write_zip_data_desc(unsigned long size,
222
unsigned long compressed_size,
223
unsigned long crc)
224
{
205
- struct zip_data_desc trailer;
206
-
207
- copy_le32(trailer.magic, 0x08074b50);
208
- copy_le32(trailer.crc32, crc);
209
- copy_le32(trailer.compressed_size, compressed_size);
210
- copy_le32(trailer.size, size);
211
- write_or_die(1, &trailer, ZIP_DATA_DESC_SIZE);
225
+ if (size >= 0xffffffff || compressed_size >= 0xffffffff) {
226
+ struct zip64_data_desc trailer;
227
+ copy_le32(trailer.magic, 0x08074b50);
228
+ copy_le32(trailer.crc32, crc);
229
+ copy_le64(trailer.compressed_size, compressed_size);
230
+ copy_le64(trailer.size, size);
231
+ write_or_die(1, &trailer, ZIP64_DATA_DESC_SIZE);
232
+ zip_offset += ZIP64_DATA_DESC_SIZE;
233
+ } else {
234
+ struct zip_data_desc trailer;
235
+ copy_le32(trailer.magic, 0x08074b50);
236
+ copy_le32(trailer.crc32, crc);
237
+ copy_le32(trailer.compressed_size, compressed_size);
238
+ copy_le32(trailer.size, size);
239
+ write_or_die(1, &trailer, ZIP_DATA_DESC_SIZE);
240
+ zip_offset += ZIP_DATA_DESC_SIZE;
241
+ }
242
}
243
244
static void set_zip_header_data_desc(struct zip_local_header *header,
@@ -252,6 +282,9 @@ static int write_zip_entry(struct archiver_args *args,
282
struct zip_local_header header;
283
uintmax_t offset = zip_offset;
284
struct zip_extra_mtime extra;
285
+ struct zip64_extra extra64;
286
+ size_t header_extra_size = ZIP_EXTRA_MTIME_SIZE;
287
+ int need_zip64_extra = 0;
288
unsigned long attr2;
289
unsigned long compressed_size;
290
unsigned long crc;
@@ -344,21 +377,40 @@ static int write_zip_entry(struct archiver_args *args,
377
extra.flags[0] = 1; /* just mtime */
378
copy_le32(extra.mtime, args->time);
379
380
+ if (size > 0xffffffff || compressed_size > 0xffffffff)
381
+ need_zip64_extra = 1;
382
+ if (stream && size > 0x7fffffff)
383
+ need_zip64_extra = 1;
384
+
385
copy_le32(header.magic, 0x04034b50);
386
copy_le16(header.version, 10);
387
copy_le16(header.flags, flags);
388
copy_le16(header.compression_method, method);
389
copy_le16(header.mtime, zip_time);
390
copy_le16(header.mdate, zip_date);
353
- set_zip_header_data_desc(&header, size, compressed_size, crc);
391
+ if (need_zip64_extra) {
392
+ set_zip_header_data_desc(&header, 0xffffffff, 0xffffffff, crc);
393
+ header_extra_size += ZIP64_EXTRA_SIZE;
394
+ } else {
395
+ set_zip_header_data_desc(&header, size, compressed_size, crc);
396
+ }
397
copy_le16(header.filename_length, pathlen);
355
- copy_le16(header.extra_length, ZIP_EXTRA_MTIME_SIZE);
398
+ copy_le16(header.extra_length, header_extra_size);
399
write_or_die(1, &header, ZIP_LOCAL_HEADER_SIZE);
400
zip_offset += ZIP_LOCAL_HEADER_SIZE;
401
write_or_die(1, path, pathlen);
402
zip_offset += pathlen;
403
write_or_die(1, &extra, ZIP_EXTRA_MTIME_SIZE);
404
zip_offset += ZIP_EXTRA_MTIME_SIZE;
405
+ if (need_zip64_extra) {
406
+ copy_le16(extra64.magic, 0x0001);
407
+ copy_le16(extra64.extra_size, ZIP64_EXTRA_PAYLOAD_SIZE);
408
+ copy_le64(extra64.size, size);
409
+ copy_le64(extra64.compressed_size, compressed_size);
410
+ write_or_die(1, &extra64, ZIP64_EXTRA_SIZE);
411
+ zip_offset += ZIP64_EXTRA_SIZE;
412
+ }
413
+
414
if (stream && method == 0) {
415
unsigned char buf[STREAM_BUFFER_SIZE];
416
ssize_t readlen;
@@ -381,7 +433,6 @@ static int write_zip_entry(struct archiver_args *args,
433
zip_offset += compressed_size;
434
435
write_zip_data_desc(size, compressed_size, crc);
384
- zip_offset += ZIP_DATA_DESC_SIZE;
436
} else if (stream && method == 8) {
437
unsigned char buf[STREAM_BUFFER_SIZE];
438
ssize_t readlen;
@@ -437,7 +488,6 @@ static int write_zip_entry(struct archiver_args *args,
488
zip_offset += compressed_size;
489
490
write_zip_data_desc(size, compressed_size, crc);
440
- zip_offset += ZIP_DATA_DESC_SIZE;
491
} else if (compressed_size > 0) {
492
write_or_die(1, out, compressed_size);
493
zip_offset += compressed_size;
@@ -446,8 +496,14 @@ static int write_zip_entry(struct archiver_args *args,
496
free(deflated);
497
free(buffer);
498
449
- if (offset > 0xffffffff) {
450
- zip64_dir_extra_payload_size += 8;
499
+ if (compressed_size > 0xffffffff || size > 0xffffffff ||
500
+ offset > 0xffffffff) {
501
+ if (compressed_size >= 0xffffffff)
502
+ zip64_dir_extra_payload_size += 8;
503
+ if (size >= 0xffffffff)
504
+ zip64_dir_extra_payload_size += 8;
505
+ if (offset >= 0xffffffff)
506
+ zip64_dir_extra_payload_size += 8;
507
zip_dir_extra_size += 2 + 2 + zip64_dir_extra_payload_size;
508
}
509
@@ -459,8 +515,8 @@ static int write_zip_entry(struct archiver_args *args,
515
strbuf_add_le(&zip_dir, 2, zip_time);
516
strbuf_add_le(&zip_dir, 2, zip_date);
517
strbuf_add_le(&zip_dir, 4, crc);
462
- strbuf_add_le(&zip_dir, 4, compressed_size);
463
- strbuf_add_le(&zip_dir, 4, size);
518
+ strbuf_add_le(&zip_dir, 4, clamp32(compressed_size));
519
+ strbuf_add_le(&zip_dir, 4, clamp32(size));
520
strbuf_add_le(&zip_dir, 2, pathlen);
521
strbuf_add_le(&zip_dir, 2, zip_dir_extra_size);
522
strbuf_add_le(&zip_dir, 2, 0); /* comment length */
@@ -473,6 +529,10 @@ static int write_zip_entry(struct archiver_args *args,
529
if (zip64_dir_extra_payload_size) {
530
strbuf_add_le(&zip_dir, 2, 0x0001); /* magic */
531
strbuf_add_le(&zip_dir, 2, zip64_dir_extra_payload_size);
532
+ if (size >= 0xffffffff)
533
+ strbuf_add_le(&zip_dir, 8, size);
534
+ if (compressed_size >= 0xffffffff)
535
+ strbuf_add_le(&zip_dir, 8, compressed_size);
536
if (offset >= 0xffffffff)
537
strbuf_add_le(&zip_dir, 8, offset);
538
}
t/t5004-archive-corner-cases.sh
+1
-1
@@ -178,7 +178,7 @@ test_expect_success EXPENSIVE,UNZIP 'zip archive bigger than 4GB' '
178
"$GIT_UNZIP" -t many-big.zip
179
'
180
181
-test_expect_failure EXPENSIVE,UNZIP,ZIPINFO 'zip archive with files bigger than 4GB' '
181
+test_expect_success EXPENSIVE,UNZIP,ZIPINFO 'zip archive with files bigger than 4GB' '
182
# Pack created with:
183
# dd if=/dev/zero of=file bs=1M count=4100 && git hash-object -w file
184
mkdir -p .git/objects/pack &&