convert: tighten the safe autocrlf handling
When a text file had been commited with CRLF and the file is commited again, the CRLF are kept if .gitattributs has "text=auto". This is done by analyzing the content of the blob stored in the index: If a '\r' is found, Git assumes that the blob was commited with CRLF. The simple search for a '\r' does not always work as expected: A file is encoded in UTF-16 with CRLF and commited. Git treats it as binary. Now the content is converted into UTF-8. At the next commit Git treats the file as text, the CRLF should be converted into LF, but isn't. Replace has_cr_in_index() with has_crlf_in_index(). When no '\r' is found, 0 is returned directly, this is the most common case. If a '\r' is found, the content is analyzed more deeply. Reported-By: Ashish Negi <ashishnegi33@gmail.com> Signed-off-by: Torsten Bögershausen <tboegi@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Torsten Bögershausen committed
Nov 26, 2017 at 13:20 UTC
86ff70a0f0d3917205e1aa82dba08db92d357cc0
2 files changed
+85
-10
convert.c
+14
-5
@@ -220,18 +220,27 @@ static void check_safe_crlf(const char *path, enum crlf_action crlf_action,
220
}
221
}
222
223
-static int has_cr_in_index(const struct index_state *istate, const char *path)
223
+static int has_crlf_in_index(const struct index_state *istate, const char *path)
224
{
225
unsigned long sz;
226
void *data;
227
- int has_cr;
227
+ const char *crp;
228
+ int has_crlf = 0;
229
230
data = read_blob_data_from_index(istate, path, &sz);
231
if (!data)
232
return 0;
232
- has_cr = memchr(data, '\r', sz) != NULL;
233
+
234
+ crp = memchr(data, '\r', sz);
235
+ if (crp) {
236
+ unsigned int ret_stats;
237
+ ret_stats = gather_convert_stats(data, sz);
238
+ if (!(ret_stats & CONVERT_STAT_BITS_BIN) &&
239
+ (ret_stats & CONVERT_STAT_BITS_TXT_CRLF))
240
+ has_crlf = 1;
241
+ }
242
free(data);
234
- return has_cr;
243
+ return has_crlf;
244
}
245
246
static int will_convert_lf_to_crlf(size_t len, struct text_stat *stats,
@@ -290,7 +299,7 @@ static int crlf_to_git(const struct index_state *istate,
299
* cherry-pick.
300
*/
301
if ((checksafe != SAFE_CRLF_RENORMALIZE) &&
293
- has_cr_in_index(istate, path))
302
+ has_crlf_in_index(istate, path))
303
convert_crlf_into_lf = 0;
304
}
305
if ((checksafe == SAFE_CRLF_WARN ||
t/t0027-auto-crlf.sh
+71
-5
@@ -43,19 +43,31 @@ create_gitattributes () {
43
} >.gitattributes
44
}
45
46
-create_NNO_files () {
46
+# Create 2 sets of files:
47
+# The NNO files are "Not NOrmalized in the repo. We use CRLF_mix_LF and store
48
+# it under different names for the different test cases, see ${pfx}
49
+# Depending on .gitattributes they are normalized at the next commit (or not)
50
+# The MIX files have different contents in the repo.
51
+# Depending on its contents, the "new safer autocrlf" may kick in.
52
+create_NNO_MIX_files () {
53
for crlf in false true input
54
do
55
for attr in "" auto text -text
56
do
57
for aeol in "" lf crlf
58
do
53
- pfx=NNO_attr_${attr}_aeol_${aeol}_${crlf}
59
+ pfx=NNO_attr_${attr}_aeol_${aeol}_${crlf} &&
60
cp CRLF_mix_LF ${pfx}_LF.txt &&
61
cp CRLF_mix_LF ${pfx}_CRLF.txt &&
62
cp CRLF_mix_LF ${pfx}_CRLF_mix_LF.txt &&
63
cp CRLF_mix_LF ${pfx}_LF_mix_CR.txt &&
58
- cp CRLF_mix_LF ${pfx}_CRLF_nul.txt
64
+ cp CRLF_mix_LF ${pfx}_CRLF_nul.txt &&
65
+ pfx=MIX_attr_${attr}_aeol_${aeol}_${crlf} &&
66
+ cp LF ${pfx}_LF.txt &&
67
+ cp CRLF ${pfx}_CRLF.txt &&
68
+ cp CRLF_mix_LF ${pfx}_CRLF_mix_LF.txt &&
69
+ cp LF_mix_CR ${pfx}_LF_mix_CR.txt &&
70
+ cp CRLF_nul ${pfx}_CRLF_nul.txt
71
done
72
done
73
done
@@ -136,6 +148,49 @@ commit_chk_wrnNNO () {
148
'
149
}
150
151
+# Commit a file with mixed line endings on top of different files
152
+# in the index. Check for warnings
153
+commit_MIX_chkwrn () {
154
+ attr=$1 ; shift
155
+ aeol=$1 ; shift
156
+ crlf=$1 ; shift
157
+ lfwarn=$1 ; shift
158
+ crlfwarn=$1 ; shift
159
+ lfmixcrlf=$1 ; shift
160
+ lfmixcr=$1 ; shift
161
+ crlfnul=$1 ; shift
162
+ pfx=MIX_attr_${attr}_aeol_${aeol}_${crlf}
163
+ #Commit file with CLRF_mix_LF on top of existing file
164
+ create_gitattributes "$attr" $aeol &&
165
+ for f in LF CRLF CRLF_mix_LF LF_mix_CR CRLF_nul
166
+ do
167
+ fname=${pfx}_$f.txt &&
168
+ cp CRLF_mix_LF $fname &&
169
+ printf Z >>"$fname" &&
170
+ git -c core.autocrlf=$crlf add $fname 2>"${pfx}_$f.err"
171
+ done
172
+
173
+ test_expect_success "commit file with mixed EOL crlf=$crlf attr=$attr LF" '
174
+ check_warning "$lfwarn" ${pfx}_LF.err
175
+ '
176
+ test_expect_success "commit file with mixed EOL attr=$attr aeol=$aeol crlf=$crlf CRLF" '
177
+ check_warning "$crlfwarn" ${pfx}_CRLF.err
178
+ '
179
+
180
+ test_expect_success "commit file with mixed EOL attr=$attr aeol=$aeol crlf=$crlf CRLF_mix_LF" '
181
+ check_warning "$lfmixcrlf" ${pfx}_CRLF_mix_LF.err
182
+ '
183
+
184
+ test_expect_success "commit file with mixed EOL attr=$attr aeol=$aeol crlf=$crlf LF_mix_cr" '
185
+ check_warning "$lfmixcr" ${pfx}_LF_mix_CR.err
186
+ '
187
+
188
+ test_expect_success "commit file with mixed EOL attr=$attr aeol=$aeol crlf=$crlf CRLF_nul" '
189
+ check_warning "$crlfnul" ${pfx}_CRLF_nul.err
190
+ '
191
+}
192
+
193
+
194
stats_ascii () {
195
case "$1" in
196
LF)
@@ -323,8 +378,8 @@ test_expect_success 'setup master' '
378
printf "\$Id: 0000000000000000000000000000000000000000 \$\r\nLINEONE\r\nLINETWO\rLINETHREE" >CRLF_mix_CR &&
379
printf "\$Id: 0000000000000000000000000000000000000000 \$\r\nLINEONEQ\r\nLINETWO\r\nLINETHREE" | q_to_nul >CRLF_nul &&
380
printf "\$Id: 0000000000000000000000000000000000000000 \$\nLINEONEQ\nLINETWO\nLINETHREE" | q_to_nul >LF_nul &&
326
- create_NNO_files CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF &&
327
- git -c core.autocrlf=false add NNO_*.txt &&
381
+ create_NNO_MIX_files CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF CRLF_mix_LF &&
382
+ git -c core.autocrlf=false add NNO_*.txt MIX_*.txt &&
383
git commit -m "mixed line endings" &&
384
test_tick
385
'
@@ -385,6 +440,17 @@ test_expect_success 'commit files attr=crlf' '
440
commit_check_warn input "crlf" "LF_CRLF" "" "LF_CRLF" "LF_CRLF" ""
441
'
442
443
+# Commit "CRLFmixLF" on top of these files already in the repo:
444
+# LF, CRLF, CRLFmixLF LF_mix_CR CRLFNULL
445
+# attr LF CRLF CRLFmixLF LF_mix_CR CRLFNUL
446
+commit_MIX_chkwrn "" "" false "" "" "" "" ""
447
+commit_MIX_chkwrn "" "" true "LF_CRLF" "" "" "LF_CRLF" "LF_CRLF"
448
+commit_MIX_chkwrn "" "" input "CRLF_LF" "" "" "CRLF_LF" "CRLF_LF"
449
+
450
+commit_MIX_chkwrn "auto" "" false "CRLF_LF" "" "" "CRLF_LF" "CRLF_LF"
451
+commit_MIX_chkwrn "auto" "" true "LF_CRLF" "" "" "LF_CRLF" "LF_CRLF"
452
+commit_MIX_chkwrn "auto" "" input "CRLF_LF" "" "" "CRLF_LF" "CRLF_LF"
453
+
454
# attr LF CRLF CRLFmixLF LF_mix_CR CRLFNUL
455
commit_chk_wrnNNO "" "" false "" "" "" "" ""
456
commit_chk_wrnNNO "" "" true LF_CRLF "" "" "" ""