@matheusbsbs / matheus-bagliano / commits / 664d44ee7f
Fork 0

userdiff: simplify word-diff safeguard

git's diff-words support has a detail that can be a little dangerous: any text not matched by a given language's tokenization pattern is treated as whitespace and changes in such text would go unnoticed. Therefore each of the built-in regexes allows a special token type consisting of a single non-whitespace character [^[:space:]]. To make sure UTF-8 sequences remain human readable, the builtin regexes also have a special token type for runs of bytes with the high bit set. In English, non-ASCII characters are usually isolated so this is analogous to the [^[:space:]] pattern, except it matches a single _multibyte_ character despite use of the C locale. Unfortunately it is easy to make typos or forget entirely to include these catch-all token types when adding support for new languages (see v1.7.3.5~16, userdiff: fix typo in ruby and python word regexes, 2010-12-18). Avoid this by including them automatically within the PATTERNS and IPATTERN macros. While at it, change the UTF-8 sequence token type to match exactly one non-ASCII multi-byte character, rather than an arbitrary run of them. Suggested-by: Thomas Rast <trast@student.ethz.ch> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Nieder committed Jan 11, 2011 at 15:48 UTC 664d44ee7fb18bdfdd66a1be760c7ee1bbe911c6
1 file changed +16 -24
userdiff.c
+16 -24
@@ -8,9 +8,11 @@ static int ndrivers;
8 static int drivers_alloc;
9
10 #define PATTERNS(name, pattern, word_regex) \
11 - { name, NULL, -1, { pattern, REG_EXTENDED }, word_regex }
11 + { name, NULL, -1, { pattern, REG_EXTENDED }, \
12 + word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" }
13 #define IPATTERN(name, pattern, word_regex) \
13 - { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, word_regex }
14 + { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, \
15 + word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" }
16 static struct userdiff_driver builtin_drivers[] = {
17 IPATTERN("fortran",
18 "!^([C*]|[ \t]*!)\n"
@@ -24,10 +26,9 @@ IPATTERN("fortran",
26 * Don't worry about format statements without leading digits since
27 * they would have been matched above as a variable anyway. */
28 "|[-+]?[0-9.]+([AaIiDdEeFfLlTtXx][Ss]?[-+]?[0-9.]*)?(_[a-zA-Z0-9][a-zA-Z0-9_]*)?"
27 - "|//|\\*\\*|::|[/<>=]="
28 - "|[^[:space:]]|[\x80-\xff]+"),
29 + "|//|\\*\\*|::|[/<>=]="),
30 PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$",
30 - "[^<>= \t]+|[^[:space:]]|[\x80-\xff]+"),
31 + "[^<>= \t]+"),
32 PATTERNS("java",
33 "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
34 "^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$",
@@ -35,8 +36,7 @@ PATTERNS("java",
36 "[a-zA-Z_][a-zA-Z0-9_]*"
37 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
38 "|[-+*/<>%&^|=!]="
38 - "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"
39 - "|[^[:space:]]|[\x80-\xff]+"),
39 + "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"),
40 PATTERNS("objc",
41 /* Negate C statements that can look like functions */
42 "!^[ \t]*(do|for|if|else|return|switch|while)\n"
@@ -49,8 +49,7 @@ PATTERNS("objc",
49 /* -- */
50 "[a-zA-Z_][a-zA-Z0-9_]*"
51 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
52 - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
53 - "|[^[:space:]]|[\x80-\xff]+"),
52 + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
53 PATTERNS("pascal",
54 "^((procedure|function|constructor|destructor|interface|"
55 "implementation|initialization|finalization)[ \t]*.*)$"
@@ -59,8 +58,7 @@ PATTERNS("pascal",
58 /* -- */
59 "[a-zA-Z_][a-zA-Z0-9_]*"
60 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
62 - "|<>|<=|>=|:=|\\.\\."
63 - "|[^[:space:]]|[\x80-\xff]+"),
61 + "|<>|<=|>=|:=|\\.\\."),
62 PATTERNS("perl",
63 "^[ \t]*package .*;\n"
64 "^[ \t]*sub .* \\{\n"
@@ -76,33 +74,29 @@ PATTERNS("perl",
74 "|&&|\\|\\||//|\\+\\+|--|\\*\\*|\\.\\.\\.?"
75 "|[-+*/%.^&<>=!|]="
76 "|=~|!~"
79 - "|<<|<>|<=>|>>"
80 - "|[^[:space:]]"),
77 + "|<<|<>|<=>|>>"),
78 PATTERNS("php",
79 "^[\t ]*(((public|protected|private|static)[\t ]+)*function.*)$\n"
80 "^[\t ]*(class.*)$",
81 /* -- */
82 "[a-zA-Z_][a-zA-Z0-9_]*"
83 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
87 - "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"
88 - "|[^[:space:]]|[\x80-\xff]+"),
84 + "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"),
85 PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$",
86 /* -- */
87 "[a-zA-Z_][a-zA-Z0-9_]*"
88 "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
93 - "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"
94 - "|[^[:space:]]|[\x80-\xff]+"),
89 + "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"),
90 /* -- */
91 PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$",
92 /* -- */
93 "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
94 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
100 - "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"
101 - "|[^[:space:]]|[\x80-\xff]+"),
95 + "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"),
96 PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
97 "[={}\"]|[^={}\" \t]+"),
98 PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
105 - "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+|[^[:space:]]"),
99 + "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+"),
100 PATTERNS("cpp",
101 /* Jump targets or access declarations */
102 "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:.*$\n"
@@ -113,8 +107,7 @@ PATTERNS("cpp",
107 /* -- */
108 "[a-zA-Z_][a-zA-Z0-9_]*"
109 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
116 - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
117 - "|[^[:space:]]|[\x80-\xff]+"),
110 + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
111 PATTERNS("csharp",
112 /* Keywords */
113 "!^[ \t]*(do|while|for|if|else|instanceof|new|return|switch|case|throw|catch|using)\n"
@@ -129,8 +122,7 @@ PATTERNS("csharp",
122 /* -- */
123 "[a-zA-Z_][a-zA-Z0-9_]*"
124 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
132 - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
133 - "|[^[:space:]]|[\x80-\xff]+"),
125 + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
126 { "default", NULL, -1, { NULL, 0 } },
127 };
128 #undef PATTERNS