Normalize text to fix french (#1504)
* Normalize text to fix french * Normalize text to fix french * Fix French? * Fix French? * Fix French? * Polyseed v0.0.5
Konstantin Ullrich committed
Jun 24, 2024 at 18:38 UTC
7dd15914d03bd4ac1a5e4c6459070fea6e6eb3e5
11 files changed
+152
-136
cw_bitcoin/lib/bitcoin_mnemonic.dart
+2
-117
@@ -2,9 +2,9 @@ import 'dart:convert';
2
import 'dart:math';
3
import 'dart:typed_data';
4
import 'package:crypto/crypto.dart';
5
-import 'package:unorm_dart/unorm_dart.dart' as unorm;
5
import 'package:cryptography/cryptography.dart' as cryptography;
6
import 'package:cw_core/sec_random_native.dart';
7
+import 'package:cw_core/utils/text_normalizer.dart';
8
9
const segwit = '100';
10
final wordlist = englishWordlist;
@@ -137,121 +137,6 @@ bool validateMnemonic(String mnemonic, {String prefix = segwit}) {
137
}
138
}
139
140
-final COMBININGCODEPOINTS = combiningcodepoints();
141
-
142
-List<int> combiningcodepoints() {
143
- final source = '300:34e|350:36f|483:487|591:5bd|5bf|5c1|5c2|5c4|5c5|5c7|610:61a|64b:65f|670|' +
144
- '6d6:6dc|6df:6e4|6e7|6e8|6ea:6ed|711|730:74a|7eb:7f3|816:819|81b:823|825:827|' +
145
- '829:82d|859:85b|8d4:8e1|8e3:8ff|93c|94d|951:954|9bc|9cd|a3c|a4d|abc|acd|b3c|' +
146
- 'b4d|bcd|c4d|c55|c56|cbc|ccd|d4d|dca|e38:e3a|e48:e4b|eb8|eb9|ec8:ecb|f18|f19|' +
147
- 'f35|f37|f39|f71|f72|f74|f7a:f7d|f80|f82:f84|f86|f87|fc6|1037|1039|103a|108d|' +
148
- '135d:135f|1714|1734|17d2|17dd|18a9|1939:193b|1a17|1a18|1a60|1a75:1a7c|1a7f|' +
149
- '1ab0:1abd|1b34|1b44|1b6b:1b73|1baa|1bab|1be6|1bf2|1bf3|1c37|1cd0:1cd2|' +
150
- '1cd4:1ce0|1ce2:1ce8|1ced|1cf4|1cf8|1cf9|1dc0:1df5|1dfb:1dff|20d0:20dc|20e1|' +
151
- '20e5:20f0|2cef:2cf1|2d7f|2de0:2dff|302a:302f|3099|309a|a66f|a674:a67d|a69e|' +
152
- 'a69f|a6f0|a6f1|a806|a8c4|a8e0:a8f1|a92b:a92d|a953|a9b3|a9c0|aab0|aab2:aab4|' +
153
- 'aab7|aab8|aabe|aabf|aac1|aaf6|abed|fb1e|fe20:fe2f|101fd|102e0|10376:1037a|' +
154
- '10a0d|10a0f|10a38:10a3a|10a3f|10ae5|10ae6|11046|1107f|110b9|110ba|11100:11102|' +
155
- '11133|11134|11173|111c0|111ca|11235|11236|112e9|112ea|1133c|1134d|11366:1136c|' +
156
- '11370:11374|11442|11446|114c2|114c3|115bf|115c0|1163f|116b6|116b7|1172b|11c3f|' +
157
- '16af0:16af4|16b30:16b36|1bc9e|1d165:1d169|1d16d:1d172|1d17b:1d182|1d185:1d18b|' +
158
- '1d1aa:1d1ad|1d242:1d244|1e000:1e006|1e008:1e018|1e01b:1e021|1e023|1e024|' +
159
- '1e026:1e02a|1e8d0:1e8d6|1e944:1e94a';
160
-
161
- return source.split('|').map((e) {
162
- if (e.contains(':')) {
163
- return e.split(':').map((hex) => int.parse(hex, radix: 16));
164
- }
165
-
166
- return int.parse(e, radix: 16);
167
- }).fold(<int>[], (List<int> acc, element) {
168
- if (element is List) {
169
- for (var i = element[0] as int; i <= (element[1] as int); i++) {}
170
- } else if (element is int) {
171
- acc.add(element);
172
- }
173
-
174
- return acc;
175
- }).toList();
176
-}
177
-
178
-String removeCombiningCharacters(String source) {
179
- return source
180
- .split('')
181
- .where((char) => !COMBININGCODEPOINTS.contains(char.codeUnits.first))
182
- .join('');
183
-}
184
-
185
-bool isCJK(String char) {
186
- final n = char.codeUnitAt(0);
187
-
188
- for (var x in CJKINTERVALS) {
189
- final imin = x[0] as num;
190
- final imax = x[1] as num;
191
-
192
- if (n >= imin && n <= imax) return true;
193
- }
194
-
195
- return false;
196
-}
197
-
198
-String removeCJKSpaces(String source) {
199
- final splitted = source.split('');
200
- final filtered = <String>[];
201
-
202
- for (var i = 0; i < splitted.length; i++) {
203
- final char = splitted[i];
204
- final isSpace = char.trim() == '';
205
- final prevIsCJK = i != 0 && isCJK(splitted[i - 1]);
206
- final nextIsCJK = i != splitted.length - 1 && isCJK(splitted[i + 1]);
207
-
208
- if (!(isSpace && prevIsCJK && nextIsCJK)) {
209
- filtered.add(char);
210
- }
211
- }
212
-
213
- return filtered.join('');
214
-}
215
-
216
-String normalizeText(String source) {
217
- final res =
218
- removeCombiningCharacters(unorm.nfkd(source).toLowerCase()).trim().split('/\s+/').join(' ');
219
-
220
- return removeCJKSpaces(res);
221
-}
222
-
223
-const CJKINTERVALS = [
224
- [0x4e00, 0x9fff, 'CJK Unified Ideographs'],
225
- [0x3400, 0x4dbf, 'CJK Unified Ideographs Extension A'],
226
- [0x20000, 0x2a6df, 'CJK Unified Ideographs Extension B'],
227
- [0x2a700, 0x2b73f, 'CJK Unified Ideographs Extension C'],
228
- [0x2b740, 0x2b81f, 'CJK Unified Ideographs Extension D'],
229
- [0xf900, 0xfaff, 'CJK Compatibility Ideographs'],
230
- [0x2f800, 0x2fa1d, 'CJK Compatibility Ideographs Supplement'],
231
- [0x3190, 0x319f, 'Kanbun'],
232
- [0x2e80, 0x2eff, 'CJK Radicals Supplement'],
233
- [0x2f00, 0x2fdf, 'CJK Radicals'],
234
- [0x31c0, 0x31ef, 'CJK Strokes'],
235
- [0x2ff0, 0x2fff, 'Ideographic Description Characters'],
236
- [0xe0100, 0xe01ef, 'Variation Selectors Supplement'],
237
- [0x3100, 0x312f, 'Bopomofo'],
238
- [0x31a0, 0x31bf, 'Bopomofo Extended'],
239
- [0xff00, 0xffef, 'Halfwidth and Fullwidth Forms'],
240
- [0x3040, 0x309f, 'Hiragana'],
241
- [0x30a0, 0x30ff, 'Katakana'],
242
- [0x31f0, 0x31ff, 'Katakana Phonetic Extensions'],
243
- [0x1b000, 0x1b0ff, 'Kana Supplement'],
244
- [0xac00, 0xd7af, 'Hangul Syllables'],
245
- [0x1100, 0x11ff, 'Hangul Jamo'],
246
- [0xa960, 0xa97f, 'Hangul Jamo Extended A'],
247
- [0xd7b0, 0xd7ff, 'Hangul Jamo Extended B'],
248
- [0x3130, 0x318f, 'Hangul Compatibility Jamo'],
249
- [0xa4d0, 0xa4ff, 'Lisu'],
250
- [0x16f00, 0x16f9f, 'Miao'],
251
- [0xa000, 0xa48f, 'Yi Syllables'],
252
- [0xa490, 0xa4cf, 'Yi Radicals'],
253
-];
254
-
140
final englishWordlist = <String>[
141
'abandon',
142
'ability',
@@ -2301,4 +2186,4 @@ final englishWordlist = <String>[
2186
'zero',
2187
'zone',
2188
'zoo'
2304
-];
\ No newline at end of file
2189
+];
cw_bitcoin/pubspec.lock
+1
-1
@@ -863,7 +863,7 @@ packages:
863
source: hosted
864
version: "1.3.2"
865
unorm_dart:
866
- dependency: "direct main"
866
+ dependency: transitive
867
description:
868
name: unorm_dart
869
sha256: "5b35bff83fce4d76467641438f9e867dc9bcfdb8c1694854f230579d68cd8f4b"
cw_bitcoin/pubspec.yaml
-1
@@ -28,7 +28,6 @@ dependencies:
28
url: https://github.com/cake-tech/bitbox-flutter.git
29
ref: Add-Support-For-OP-Return-data
30
rxdart: ^0.27.5
31
- unorm_dart: ^0.2.0
31
cryptography: ^2.0.5
32
bitcoin_base:
33
git:
cw_core/lib/utils/text_normalizer.dart
new
+117
@@ -0,0 +1,117 @@
1
+import 'package:unorm_dart/unorm_dart.dart' as unorm;
2
+
3
+const CJKINTERVALS = [
4
+ [0x4e00, 0x9fff, 'CJK Unified Ideographs'],
5
+ [0x3400, 0x4dbf, 'CJK Unified Ideographs Extension A'],
6
+ [0x20000, 0x2a6df, 'CJK Unified Ideographs Extension B'],
7
+ [0x2a700, 0x2b73f, 'CJK Unified Ideographs Extension C'],
8
+ [0x2b740, 0x2b81f, 'CJK Unified Ideographs Extension D'],
9
+ [0xf900, 0xfaff, 'CJK Compatibility Ideographs'],
10
+ [0x2f800, 0x2fa1d, 'CJK Compatibility Ideographs Supplement'],
11
+ [0x3190, 0x319f, 'Kanbun'],
12
+ [0x2e80, 0x2eff, 'CJK Radicals Supplement'],
13
+ [0x2f00, 0x2fdf, 'CJK Radicals'],
14
+ [0x31c0, 0x31ef, 'CJK Strokes'],
15
+ [0x2ff0, 0x2fff, 'Ideographic Description Characters'],
16
+ [0xe0100, 0xe01ef, 'Variation Selectors Supplement'],
17
+ [0x3100, 0x312f, 'Bopomofo'],
18
+ [0x31a0, 0x31bf, 'Bopomofo Extended'],
19
+ [0xff00, 0xffef, 'Halfwidth and Fullwidth Forms'],
20
+ [0x3040, 0x309f, 'Hiragana'],
21
+ [0x30a0, 0x30ff, 'Katakana'],
22
+ [0x31f0, 0x31ff, 'Katakana Phonetic Extensions'],
23
+ [0x1b000, 0x1b0ff, 'Kana Supplement'],
24
+ [0xac00, 0xd7af, 'Hangul Syllables'],
25
+ [0x1100, 0x11ff, 'Hangul Jamo'],
26
+ [0xa960, 0xa97f, 'Hangul Jamo Extended A'],
27
+ [0xd7b0, 0xd7ff, 'Hangul Jamo Extended B'],
28
+ [0x3130, 0x318f, 'Hangul Compatibility Jamo'],
29
+ [0xa4d0, 0xa4ff, 'Lisu'],
30
+ [0x16f00, 0x16f9f, 'Miao'],
31
+ [0xa000, 0xa48f, 'Yi Syllables'],
32
+ [0xa490, 0xa4cf, 'Yi Radicals'],
33
+];
34
+
35
+final COMBININGCODEPOINTS = combiningcodepoints();
36
+
37
+List<int> combiningcodepoints() {
38
+ final source = '300:34e|350:36f|483:487|591:5bd|5bf|5c1|5c2|5c4|5c5|5c7|610:61a|64b:65f|670|' +
39
+ '6d6:6dc|6df:6e4|6e7|6e8|6ea:6ed|711|730:74a|7eb:7f3|816:819|81b:823|825:827|' +
40
+ '829:82d|859:85b|8d4:8e1|8e3:8ff|93c|94d|951:954|9bc|9cd|a3c|a4d|abc|acd|b3c|' +
41
+ 'b4d|bcd|c4d|c55|c56|cbc|ccd|d4d|dca|e38:e3a|e48:e4b|eb8|eb9|ec8:ecb|f18|f19|' +
42
+ 'f35|f37|f39|f71|f72|f74|f7a:f7d|f80|f82:f84|f86|f87|fc6|1037|1039|103a|108d|' +
43
+ '135d:135f|1714|1734|17d2|17dd|18a9|1939:193b|1a17|1a18|1a60|1a75:1a7c|1a7f|' +
44
+ '1ab0:1abd|1b34|1b44|1b6b:1b73|1baa|1bab|1be6|1bf2|1bf3|1c37|1cd0:1cd2|' +
45
+ '1cd4:1ce0|1ce2:1ce8|1ced|1cf4|1cf8|1cf9|1dc0:1df5|1dfb:1dff|20d0:20dc|20e1|' +
46
+ '20e5:20f0|2cef:2cf1|2d7f|2de0:2dff|302a:302f|3099|309a|a66f|a674:a67d|a69e|' +
47
+ 'a69f|a6f0|a6f1|a806|a8c4|a8e0:a8f1|a92b:a92d|a953|a9b3|a9c0|aab0|aab2:aab4|' +
48
+ 'aab7|aab8|aabe|aabf|aac1|aaf6|abed|fb1e|fe20:fe2f|101fd|102e0|10376:1037a|' +
49
+ '10a0d|10a0f|10a38:10a3a|10a3f|10ae5|10ae6|11046|1107f|110b9|110ba|11100:11102|' +
50
+ '11133|11134|11173|111c0|111ca|11235|11236|112e9|112ea|1133c|1134d|11366:1136c|' +
51
+ '11370:11374|11442|11446|114c2|114c3|115bf|115c0|1163f|116b6|116b7|1172b|11c3f|' +
52
+ '16af0:16af4|16b30:16b36|1bc9e|1d165:1d169|1d16d:1d172|1d17b:1d182|1d185:1d18b|' +
53
+ '1d1aa:1d1ad|1d242:1d244|1e000:1e006|1e008:1e018|1e01b:1e021|1e023|1e024|' +
54
+ '1e026:1e02a|1e8d0:1e8d6|1e944:1e94a';
55
+
56
+ return source.split('|').map((e) {
57
+ if (e.contains(':')) {
58
+ return e.split(':').map((hex) => int.parse(hex, radix: 16));
59
+ }
60
+
61
+ return int.parse(e, radix: 16);
62
+ }).fold(<int>[], (List<int> acc, element) {
63
+ if (element is List) {
64
+ for (var i = element[0] as int; i <= (element[1] as int); i++) {}
65
+ } else if (element is int) {
66
+ acc.add(element);
67
+ }
68
+
69
+ return acc;
70
+ }).toList();
71
+}
72
+
73
+String _removeCombiningCharacters(String source) {
74
+ return source
75
+ .split('')
76
+ .where((char) => !COMBININGCODEPOINTS.contains(char.codeUnits.first))
77
+ .join('');
78
+}
79
+
80
+String _removeCJKSpaces(String source) {
81
+ final splitted = source.split('');
82
+ final filtered = <String>[];
83
+
84
+ for (var i = 0; i < splitted.length; i++) {
85
+ final char = splitted[i];
86
+ final isSpace = char.trim() == '';
87
+ final prevIsCJK = i != 0 && _isCJK(splitted[i - 1]);
88
+ final nextIsCJK = i != splitted.length - 1 && _isCJK(splitted[i + 1]);
89
+
90
+ if (!(isSpace && prevIsCJK && nextIsCJK)) {
91
+ filtered.add(char);
92
+ }
93
+ }
94
+
95
+ return filtered.join('');
96
+}
97
+
98
+bool _isCJK(String char) {
99
+ final n = char.codeUnitAt(0);
100
+
101
+ for (var x in CJKINTERVALS) {
102
+ final imin = x[0] as num;
103
+ final imax = x[1] as num;
104
+
105
+ if (n >= imin && n <= imax) return true;
106
+ }
107
+
108
+ return false;
109
+}
110
+
111
+/// This method normalize text which transforms Unicode text into an equivalent decomposed form, allowing for easier sorting and searching of text.
112
+String normalizeText(String source) {
113
+ final res =
114
+ _removeCombiningCharacters(unorm.nfkd(source).toLowerCase()).trim().split('/\s+/').join(' ');
115
+
116
+ return _removeCJKSpaces(res);
117
+}
cw_core/pubspec.lock
+8
@@ -656,6 +656,14 @@ packages:
656
url: "https://pub.dev"
657
source: hosted
658
version: "1.3.2"
659
+ unorm_dart:
660
+ dependency: "direct main"
661
+ description:
662
+ name: unorm_dart
663
+ sha256: "5b35bff83fce4d76467641438f9e867dc9bcfdb8c1694854f230579d68cd8f4b"
664
+ url: "https://pub.dev"
665
+ source: hosted
666
+ version: "0.2.0"
667
vector_math:
668
dependency: transitive
669
description:
cw_core/pubspec.yaml
+1
@@ -20,6 +20,7 @@ dependencies:
20
intl: ^0.18.0
21
encrypt: ^5.0.1
22
socks5_proxy: ^1.0.4
23
+ unorm_dart: ^0.3.0
24
# tor:
25
# git:
26
# url: https://github.com/cake-tech/tor.git
cw_monero/pubspec.yaml
+1
-1
@@ -19,7 +19,7 @@ dependencies:
19
flutter_mobx: ^2.0.6+1
20
intl: ^0.18.0
21
encrypt: ^5.0.1
22
- polyseed: ^0.0.2
22
+ polyseed: ^0.0.5
23
cw_core:
24
path: ../cw_core
25
lib/core/seed_validator.dart
+9
-6
@@ -1,15 +1,15 @@
1
import 'package:cake_wallet/bitcoin/bitcoin.dart';
2
-import 'package:cake_wallet/ethereum/ethereum.dart';
3
-import 'package:cake_wallet/haven/haven.dart';
2
import 'package:cake_wallet/core/validator.dart';
3
import 'package:cake_wallet/entities/mnemonic_item.dart';
4
+import 'package:cake_wallet/ethereum/ethereum.dart';
5
+import 'package:cake_wallet/haven/haven.dart';
6
+import 'package:cake_wallet/monero/monero.dart';
7
+import 'package:cake_wallet/nano/nano.dart';
8
import 'package:cake_wallet/polygon/polygon.dart';
9
import 'package:cake_wallet/solana/solana.dart';
10
import 'package:cake_wallet/tron/tron.dart';
9
-import 'package:cw_core/wallet_type.dart';
10
-import 'package:cake_wallet/monero/monero.dart';
11
-import 'package:cake_wallet/nano/nano.dart';
11
import 'package:cake_wallet/utils/language_list.dart';
12
+import 'package:cw_core/wallet_type.dart';
13
14
class SeedValidator extends Validator<MnemonicItem> {
15
SeedValidator({required this.type, required this.language})
@@ -41,13 +41,16 @@ class SeedValidator extends Validator<MnemonicItem> {
41
return polygon!.getPolygonWordList(language);
42
case WalletType.solana:
43
return solana!.getSolanaWordList(language);
44
- case WalletType.tron:
44
+ case WalletType.tron:
45
return tron!.getTronWordList(language);
46
default:
47
return [];
48
}
49
}
50
51
+ static bool needsNormalization(String language) =>
52
+ ["POLYSEED_French", "POLYSEED_Spanish"].contains(language);
53
+
54
static List<String> getBitcoinWordList(String language) {
55
assert(language.toLowerCase() == LanguageList.english.toLowerCase());
56
return bitcoin!.getWordList();
lib/src/widgets/seed_widget.dart
+6
-4
@@ -1,11 +1,11 @@
1
+import 'package:cake_wallet/core/seed_validator.dart';
2
import 'package:cake_wallet/generated/i18n.dart';
3
+import 'package:cake_wallet/src/widgets/validable_annotated_editable_text.dart';
4
import 'package:cake_wallet/themes/extensions/cake_text_theme.dart';
5
+import 'package:cake_wallet/themes/extensions/send_page_theme.dart';
6
import 'package:cw_core/wallet_type.dart';
4
-import 'package:cake_wallet/src/widgets/validable_annotated_editable_text.dart';
7
import 'package:flutter/material.dart';
8
import 'package:flutter/services.dart';
7
-import 'package:cake_wallet/core/seed_validator.dart';
8
-import 'package:cake_wallet/themes/extensions/send_page_theme.dart';
9
10
class SeedWidget extends StatefulWidget {
11
SeedWidget({
@@ -23,7 +23,6 @@ class SeedWidget extends StatefulWidget {
23
}
24
25
class SeedWidgetState extends State<SeedWidget> {
26
-
26
SeedWidgetState(String language, this.type)
27
: controller = TextEditingController(),
28
focusNode = FocusNode(),
@@ -46,6 +45,7 @@ class SeedWidgetState extends State<SeedWidget> {
45
final FocusNode focusNode;
46
final WalletType type;
47
List<String> words;
48
+ bool normalizeSeed = false;
49
bool _showPlaceholder;
50
51
String get text => controller.text;
@@ -60,6 +60,7 @@ class SeedWidgetState extends State<SeedWidget> {
60
void changeSeedLanguage(String language) {
61
setState(() {
62
words = SeedValidator.getWordList(type: type, language: language);
63
+ normalizeSeed = SeedValidator.needsNormalization(language);
64
});
65
}
66
@@ -97,6 +98,7 @@ class SeedWidgetState extends State<SeedWidget> {
98
focusNode: focusNode,
99
controller: controller,
100
words: words,
101
+ normalizeSeed: normalizeSeed,
102
textStyle: TextStyle(
103
color: Theme.of(context).extension<CakeTextTheme>()!.titleColor,
104
backgroundColor: Colors.transparent,
lib/src/widgets/validable_annotated_editable_text.dart
+6
-3
@@ -1,6 +1,6 @@
1
+import 'package:cw_core/utils/text_normalizer.dart';
2
import 'package:flutter/material.dart';
3
3
-
4
extension Compare<T> on Comparable<T> {
5
bool operator <=(T other) => compareTo(other) <= 0;
6
bool operator >=(T other) => compareTo(other) >= 0;
@@ -39,6 +39,7 @@ class ValidatableAnnotatedEditableText extends EditableText {
39
required this.validStyle,
40
required this.invalidStyle,
41
required this.words,
42
+ this.normalizeSeed = false,
43
TextStyle textStyle = const TextStyle(
44
color: Colors.black,
45
backgroundColor: Colors.transparent,
@@ -74,6 +75,7 @@ class ValidatableAnnotatedEditableText extends EditableText {
75
showSelectionHandles: true,
76
showCursor: true);
77
78
+ final bool normalizeSeed;
79
final List<String> words;
80
final TextStyle validStyle;
81
final TextStyle invalidStyle;
@@ -137,7 +139,8 @@ class ValidatableAnnotatedEditableTextState extends EditableTextState {
139
return result;
140
}
141
140
- bool validate(String source) => widget.words.indexOf(source) >= 0;
142
+ bool validate(String source) =>
143
+ widget.words.indexOf(widget.normalizeSeed ? normalizeText(source) : source) >= 0;
144
145
List<TextRange> range(String pattern, String source) {
146
final result = <TextRange>[];
@@ -173,4 +176,4 @@ class ValidatableAnnotatedEditableTextState extends EditableTextState {
176
177
return TextSpan(style: widget.style, text: text);
178
}
176
-}
\ No newline at end of file
179
+}
pubspec_base.yaml
+1
-3
@@ -60,8 +60,6 @@ dependencies:
60
git:
61
url: https://github.com/cake-tech/flutter_file_picker.git
62
ref: master
63
- unorm_dart: ^0.2.0
64
- # check unorm_dart for usage and for replace
63
permission_handler: ^10.0.0
64
device_display_brightness:
65
git:
@@ -100,7 +98,7 @@ dependencies:
98
# ref: main
99
socks5_proxy: ^1.0.4
100
flutter_svg: ^2.0.9
103
- polyseed: ^0.0.4
101
+ polyseed: ^0.0.5
102
nostr_tools: ^1.0.9
103
solana: ^0.30.1
104
bitcoin_base: