t/lib-unicode-nfc-nfd: helper prereqs for testing unicode nfc/nfd
Create a set of prereqs to help understand how file names are handled by the filesystem when they contain NFC and NFD Unicode characters. Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff Hostetler committed
May 26, 2022 at 21:47 UTC
00991e1013750130bbe4b4ac81279d6997935236
1 file changed
+162
t/lib-unicode-nfc-nfd.sh
new
+162
@@ -0,0 +1,162 @@
1
+# Help detect how Unicode NFC and NFD are handled on the filesystem.
2
+
3
+# A simple character that has a NFD form.
4
+#
5
+# NFC: U+00e9 LATIN SMALL LETTER E WITH ACUTE
6
+# UTF8(NFC): \xc3 \xa9
7
+#
8
+# NFD: U+0065 LATIN SMALL LETTER E
9
+# U+0301 COMBINING ACUTE ACCENT
10
+# UTF8(NFD): \x65 + \xcc \x81
11
+#
12
+utf8_nfc=$(printf "\xc3\xa9")
13
+utf8_nfd=$(printf "\x65\xcc\x81")
14
+
15
+# Is the OS or the filesystem "Unicode composition sensitive"?
16
+#
17
+# That is, does the OS or the filesystem allow files to exist with
18
+# both the NFC and NFD spellings? Or, does the OS/FS lie to us and
19
+# tell us that the NFC and NFD forms are equivalent.
20
+#
21
+# This is or may be independent of what type of filesystem we have,
22
+# since it might be handled by the OS at a layer above the FS.
23
+# Testing shows on MacOS using APFS, HFS+, and FAT32 reports a
24
+# collision, for example.
25
+#
26
+# This does not tell us how the Unicode pathname will be spelled
27
+# on disk, but rather only that the two spelling "collide". We
28
+# will examine the actual on disk spelling in a later prereq.
29
+#
30
+test_lazy_prereq UNICODE_COMPOSITION_SENSITIVE '
31
+ mkdir trial_${utf8_nfc} &&
32
+ mkdir trial_${utf8_nfd}
33
+'
34
+
35
+# Is the spelling of an NFC pathname preserved on disk?
36
+#
37
+# On MacOS with HFS+ and FAT32, NFC paths are converted into NFD
38
+# and on APFS, NFC paths are preserved. As we have established
39
+# above, this is independent of "composition sensitivity".
40
+#
41
+test_lazy_prereq UNICODE_NFC_PRESERVED '
42
+ mkdir c_${utf8_nfc} &&
43
+ ls | test-tool hexdump >dump &&
44
+ grep "63 5f c3 a9" dump
45
+'
46
+
47
+# Is the spelling of an NFD pathname preserved on disk?
48
+#
49
+test_lazy_prereq UNICODE_NFD_PRESERVED '
50
+ mkdir d_${utf8_nfd} &&
51
+ ls | test-tool hexdump >dump &&
52
+ grep "64 5f 65 cc 81" dump
53
+'
54
+
55
+# The following _DOUBLE_ forms are more for my curiosity,
56
+# but there may be quirks lurking when there are multiple
57
+# combining characters in non-canonical order.
58
+
59
+# Unicode also allows multiple combining characters
60
+# that can be decomposed in pieces.
61
+#
62
+# NFC: U+1f67 GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI
63
+# UTF8(NFC): \xe1 \xbd \xa7
64
+#
65
+# NFD1: U+1f61 GREEK SMALL LETTER OMEGA WITH DASIA
66
+# U+0342 COMBINING GREEK PERISPOMENI
67
+# UTF8(NFD1): \xe1 \xbd \xa1 + \xcd \x82
68
+#
69
+# But U+1f61 decomposes into
70
+# NFD2: U+03c9 GREEK SMALL LETTER OMEGA
71
+# U+0314 COMBINING REVERSED COMMA ABOVE
72
+# UTF8(NFD2): \xcf \x89 + \xcc \x94
73
+#
74
+# Yielding: \xcf \x89 + \xcc \x94 + \xcd \x82
75
+#
76
+# Note that I've used the canonical ordering of the
77
+# combinining characters. It is also possible to
78
+# swap them. My testing shows that that non-standard
79
+# ordering also causes a collision in mkdir. However,
80
+# the resulting names don't draw correctly on the
81
+# terminal (implying that the on-disk format also has
82
+# them out of order).
83
+#
84
+greek_nfc=$(printf "\xe1\xbd\xa7")
85
+greek_nfd1=$(printf "\xe1\xbd\xa1\xcd\x82")
86
+greek_nfd2=$(printf "\xcf\x89\xcc\x94\xcd\x82")
87
+
88
+# See if a double decomposition also collides.
89
+#
90
+test_lazy_prereq UNICODE_DOUBLE_COMPOSITION_SENSITIVE '
91
+ mkdir trial_${greek_nfc} &&
92
+ mkdir trial_${greek_nfd2}
93
+'
94
+
95
+# See if the NFC spelling appears on the disk.
96
+#
97
+test_lazy_prereq UNICODE_DOUBLE_NFC_PRESERVED '
98
+ mkdir c_${greek_nfc} &&
99
+ ls | test-tool hexdump >dump &&
100
+ grep "63 5f e1 bd a7" dump
101
+'
102
+
103
+# See if the NFD spelling appears on the disk.
104
+#
105
+test_lazy_prereq UNICODE_DOUBLE_NFD_PRESERVED '
106
+ mkdir d_${greek_nfd2} &&
107
+ ls | test-tool hexdump >dump &&
108
+ grep "64 5f cf 89 cc 94 cd 82" dump
109
+'
110
+
111
+# The following is for debugging. I found it useful when
112
+# trying to understand the various (OS, FS) quirks WRT
113
+# Unicode and how composition/decomposition is handled.
114
+# For example, when trying to understand how (macOS, APFS)
115
+# and (macOS, HFS) and (macOS, FAT32) compare.
116
+#
117
+# It is rather noisy, so it is disabled by default.
118
+#
119
+if test "$unicode_debug" = "true"
120
+then
121
+ if test_have_prereq UNICODE_COMPOSITION_SENSITIVE
122
+ then
123
+ echo NFC and NFD are distinct on this OS/filesystem.
124
+ else
125
+ echo NFC and NFD are aliases on this OS/filesystem.
126
+ fi
127
+
128
+ if test_have_prereq UNICODE_NFC_PRESERVED
129
+ then
130
+ echo NFC maintains original spelling.
131
+ else
132
+ echo NFC is modified.
133
+ fi
134
+
135
+ if test_have_prereq UNICODE_NFD_PRESERVED
136
+ then
137
+ echo NFD maintains original spelling.
138
+ else
139
+ echo NFD is modified.
140
+ fi
141
+
142
+ if test_have_prereq UNICODE_DOUBLE_COMPOSITION_SENSITIVE
143
+ then
144
+ echo DOUBLE NFC and NFD are distinct on this OS/filesystem.
145
+ else
146
+ echo DOUBLE NFC and NFD are aliases on this OS/filesystem.
147
+ fi
148
+
149
+ if test_have_prereq UNICODE_DOUBLE_NFC_PRESERVED
150
+ then
151
+ echo Double NFC maintains original spelling.
152
+ else
153
+ echo Double NFC is modified.
154
+ fi
155
+
156
+ if test_have_prereq UNICODE_DOUBLE_NFD_PRESERVED
157
+ then
158
+ echo Double NFD maintains original spelling.
159
+ else
160
+ echo Double NFD is modified.
161
+ fi
162
+fi