commit: reject invalid UTF-8 codepoints
The commit code already contains code for validating UTF-8, but it does not check for invalid values, such as guaranteed non-characters and surrogates. Fix this by explicitly checking for and rejecting such characters. Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
brian m. carlson committed
Jul 4, 2013 at 17:19 UTC
28110d4bfc613d875231d3d7788500d876136ffc
2 files changed
+30
-5
commit.c
+22
-5
@@ -1244,6 +1244,7 @@ static int find_invalid_utf8(const char *buf, int len)
1244
while (len) {
1245
unsigned char c = *buf++;
1246
int bytes, bad_offset;
1247
+ unsigned int codepoint;
1248
1249
len--;
1250
offset++;
@@ -1264,24 +1265,40 @@ static int find_invalid_utf8(const char *buf, int len)
1265
bytes++;
1266
}
1267
1267
- /* Must be between 1 and 5 more bytes */
1268
- if (bytes < 1 || bytes > 5)
1268
+ /*
1269
+ * Must be between 1 and 3 more bytes. Longer sequences result in
1270
+ * codepoints beyond U+10FFFF, which are guaranteed never to exist.
1271
+ */
1272
+ if (bytes < 1 || 3 < bytes)
1273
return bad_offset;
1274
1275
/* Do we *have* that many bytes? */
1276
if (len < bytes)
1277
return bad_offset;
1278
1279
+ /* Place the encoded bits at the bottom of the value. */
1280
+ codepoint = (c & 0x7f) >> bytes;
1281
+
1282
offset += bytes;
1283
len -= bytes;
1284
1285
/* And verify that they are good continuation bytes */
1286
do {
1287
+ codepoint <<= 6;
1288
+ codepoint |= *buf & 0x3f;
1289
if ((*buf++ & 0xc0) != 0x80)
1290
return bad_offset;
1291
} while (--bytes);
1292
1284
- /* We could/should check the value and length here too */
1293
+ /* No codepoints can ever be allocated beyond U+10FFFF. */
1294
+ if (codepoint > 0x10ffff)
1295
+ return bad_offset;
1296
+ /* Surrogates are only for UTF-16 and cannot be encoded in UTF-8. */
1297
+ if ((codepoint & 0x1ff800) == 0xd800)
1298
+ return bad_offset;
1299
+ /* U+FFFE and U+FFFF are guaranteed non-characters. */
1300
+ if ((codepoint & 0x1ffffe) == 0xfffe)
1301
+ return bad_offset;
1302
}
1303
return -1;
1304
}
@@ -1292,8 +1309,8 @@ static int find_invalid_utf8(const char *buf, int len)
1309
* If it isn't, it assumes any non-utf8 characters are Latin1,
1310
* and does the conversion.
1311
*
1295
- * Fixme: we should probably also disallow overlong forms and
1296
- * invalid characters. But we don't do that currently.
1312
+ * Fixme: we should probably also disallow overlong forms.
1313
+ * But we don't do that currently.
1314
*/
1315
static int verify_utf8(struct strbuf *buf)
1316
{
t/t3900-i18n-commit.sh
+8
@@ -39,6 +39,14 @@ test_expect_failure 'UTF-16 refused because of NULs' '
39
git commit -a -F "$TEST_DIRECTORY"/t3900/UTF-16.txt
40
'
41
42
+test_expect_success 'UTF-8 invalid characters refused' '
43
+ test_when_finished "rm -f $HOME/stderr $HOME/invalid" &&
44
+ echo "UTF-8 characters" >F &&
45
+ printf "Commit message\n\nInvalid surrogate:\355\240\200\n" \
46
+ >"$HOME/invalid" &&
47
+ git commit -a -F "$HOME/invalid" 2>"$HOME"/stderr &&
48
+ grep "did not conform" "$HOME"/stderr
49
+'
50
51
for H in ISO8859-1 eucJP ISO-2022-JP
52
do