diff --stat: mark any file larger than core.bigfilethreshold binary
Too large files may lead to failure to allocate memory. If it happens here, it could impact quite a few commands that involve diff. Moreover, too large files are inefficient to compare anyway (and most likely non-text), so mark them binary and skip looking at their content. Noticed-by: Dale R. Worley <worley@alum.mit.edu> Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Aug 16, 2014 at 10:08 UTC
6bf3b813486b4528feca39d599c256f662defc14
5 files changed
+27
-11
Documentation/config.txt
+2
-1
@@ -499,7 +499,8 @@ core.bigFileThreshold::
499
Files larger than this size are stored deflated, without
500
attempting delta compression. Storing large files without
501
delta compression avoids excessive memory usage, at the
502
- slight expense of increased disk usage.
502
+ slight expense of increased disk usage. Additionally files
503
+ larger than this size are always treated as binary.
504
+
505
Default is 512 MiB on all platforms. This should be reasonable
506
for most projects as source code and other text files can still
Documentation/gitattributes.txt
+2
-2
@@ -440,8 +440,8 @@ Unspecified::
440
441
A path to which the `diff` attribute is unspecified
442
first gets its contents inspected, and if it looks like
443
- text, it is treated as text. Otherwise it would
444
- generate `Binary files differ`.
443
+ text and is smaller than core.bigFileThreshold, it is treated
444
+ as text. Otherwise it would generate `Binary files differ`.
445
446
String::
447
diff.c
+18
-8
@@ -2188,8 +2188,8 @@ int diff_filespec_is_binary(struct diff_filespec *one)
2188
one->is_binary = one->driver->binary;
2189
else {
2190
if (!one->data && DIFF_FILE_VALID(one))
2191
- diff_populate_filespec(one, 0);
2192
- if (one->data)
2191
+ diff_populate_filespec(one, CHECK_BINARY);
2192
+ if (one->is_binary == -1 && one->data)
2193
one->is_binary = buffer_is_binary(one->data,
2194
one->size);
2195
if (one->is_binary == -1)
@@ -2725,6 +2725,11 @@ int diff_populate_filespec(struct diff_filespec *s, unsigned int flags)
2725
}
2726
if (size_only)
2727
return 0;
2728
+ if ((flags & CHECK_BINARY) &&
2729
+ s->size > big_file_threshold && s->is_binary == -1) {
2730
+ s->is_binary = 1;
2731
+ return 0;
2732
+ }
2733
fd = open(s->path, O_RDONLY);
2734
if (fd < 0)
2735
goto err_empty;
@@ -2746,16 +2751,21 @@ int diff_populate_filespec(struct diff_filespec *s, unsigned int flags)
2751
}
2752
else {
2753
enum object_type type;
2749
- if (size_only) {
2754
+ if (size_only || (flags & CHECK_BINARY)) {
2755
type = sha1_object_info(s->sha1, &s->size);
2756
if (type < 0)
2757
die("unable to read %s", sha1_to_hex(s->sha1));
2753
- } else {
2754
- s->data = read_sha1_file(s->sha1, &type, &s->size);
2755
- if (!s->data)
2756
- die("unable to read %s", sha1_to_hex(s->sha1));
2757
- s->should_free = 1;
2758
+ if (size_only)
2759
+ return 0;
2760
+ if (s->size > big_file_threshold && s->is_binary == -1) {
2761
+ s->is_binary = 1;
2762
+ return 0;
2763
+ }
2764
}
2765
+ s->data = read_sha1_file(s->sha1, &type, &s->size);
2766
+ if (!s->data)
2767
+ die("unable to read %s", sha1_to_hex(s->sha1));
2768
+ s->should_free = 1;
2769
}
2770
return 0;
2771
}
diffcore.h
+1
@@ -56,6 +56,7 @@ extern void fill_filespec(struct diff_filespec *, const unsigned char *,
56
int, unsigned short);
57
58
#define CHECK_SIZE_ONLY 1
59
+#define CHECK_BINARY 2
60
extern int diff_populate_filespec(struct diff_filespec *, unsigned int);
61
extern void diff_free_filespec_data(struct diff_filespec *);
62
extern void diff_free_filespec_blob(struct diff_filespec *);
t/t1050-large.sh
+4
@@ -112,6 +112,10 @@ test_expect_success 'diff --raw' '
112
git diff --raw HEAD^
113
'
114
115
+test_expect_success 'diff --stat' '
116
+ git diff --stat HEAD^ HEAD
117
+'
118
+
119
test_expect_success 'hash-object' '
120
git hash-object large1
121
'