@tduncan / Cortana
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
dart 331 lines 4.81 KB
Raw
1 import 'dart:convert';
2
3 class SimpleStringSink implements Sink<String> {
4 final void Function(String) onAdd;
5 SimpleStringSink(this.onAdd);
6
7 @override
8 void add(String data) => onAdd(data);
9
10 @override
11 void close() {}
12 }
13
14 /// A decoder that converts BPE-encoded unicode strings (used by Qwen/GPT-2 tokenizers)
15 /// back into standard UTF-8 strings. Supports streaming/chunked decoding.
16 class BpeDecoder {
17 late final ByteConversionSink _utf8Sink;
18 String _decoded = '';
19 bool _hasError = false;
20
21 BpeDecoder() {
22 _utf8Sink = utf8.decoder.startChunkedConversion(
23 SimpleStringSink((str) {
24 _decoded += str;
25 }),
26 );
27 }
28
29 /// Decodes a chunk of a BPE-encoded unicode string and returns the newly decoded text segment.
30 String decodeChunk(String bpeChunk) {
31 if (bpeChunk.isEmpty) return '';
32 if (_hasError) return bpeChunk;
33
34 // Check if bpeChunk contains characters with code point > 255 that are NOT in _bpeToByte.
35 // If so, the input is ALREADY a decoded standard string (e.g. Turkish 'İ' = 352).
36 for (var i = 0; i < bpeChunk.length; i++) {
37 final code = bpeChunk.codeUnitAt(i);
38 if (code > 255 && !_bpeToByte.containsKey(code)) {
39 return bpeChunk;
40 }
41 }
42
43 final bytes = <int>[];
44 for (var i = 0; i < bpeChunk.length; i++) {
45 final code = bpeChunk.codeUnitAt(i);
46 final byte = _bpeToByte[code] ?? code;
47 if (byte >= 0 && byte <= 255) {
48 bytes.add(byte);
49 }
50 }
51
52 final previousLength = _decoded.length;
53 try {
54 _utf8Sink.add(bytes);
55 return _decoded.substring(previousLength);
56 } catch (_) {
57 _hasError = true;
58 return bpeChunk;
59 }
60 }
61
62 /// Flushes any remaining bytes in the decoder buffer and returns the remaining decoded text.
63 String flush() {
64 if (_hasError) return '';
65 final previousLength = _decoded.length;
66 try {
67 _utf8Sink.close();
68 return _decoded.substring(previousLength);
69 } catch (_) {
70 return '';
71 }
72 }
73 }
74
75 const Map<int, int> _bpeToByte = {
76 33: 33,
77 34: 34,
78 35: 35,
79 36: 36,
80 37: 37,
81 38: 38,
82 39: 39,
83 40: 40,
84 41: 41,
85 42: 42,
86 43: 43,
87 44: 44,
88 45: 45,
89 46: 46,
90 47: 47,
91 48: 48,
92 49: 49,
93 50: 50,
94 51: 51,
95 52: 52,
96 53: 53,
97 54: 54,
98 55: 55,
99 56: 56,
100 57: 57,
101 58: 58,
102 59: 59,
103 60: 60,
104 61: 61,
105 62: 62,
106 63: 63,
107 64: 64,
108 65: 65,
109 66: 66,
110 67: 67,
111 68: 68,
112 69: 69,
113 70: 70,
114 71: 71,
115 72: 72,
116 73: 73,
117 74: 74,
118 75: 75,
119 76: 76,
120 77: 77,
121 78: 78,
122 79: 79,
123 80: 80,
124 81: 81,
125 82: 82,
126 83: 83,
127 84: 84,
128 85: 85,
129 86: 86,
130 87: 87,
131 88: 88,
132 90: 90,
133 91: 91,
134 92: 92,
135 93: 93,
136 94: 94,
137 95: 95,
138 96: 96,
139 97: 97,
140 98: 98,
141 99: 99,
142 100: 100,
143 101: 101,
144 102: 102,
145 103: 103,
146 104: 104,
147 105: 105,
148 106: 106,
149 107: 107,
150 108: 108,
151 109: 109,
152 110: 110,
153 111: 111,
154 112: 112,
155 113: 113,
156 114: 114,
157 115: 115,
158 116: 116,
159 117: 117,
160 118: 118,
161 119: 119,
162 120: 120,
163 121: 121,
164 122: 122,
165 123: 123,
166 124: 124,
167 125: 125,
168 126: 126,
169 161: 161,
170 162: 162,
171 163: 163,
172 164: 164,
173 165: 165,
174 166: 166,
175 167: 167,
176 168: 168,
177 169: 169,
178 170: 170,
179 171: 171,
180 172: 172,
181 174: 174,
182 175: 175,
183 176: 176,
184 177: 177,
185 178: 178,
186 179: 179,
187 180: 180,
188 181: 181,
189 182: 182,
190 183: 183,
191 184: 184,
192 185: 185,
193 186: 186,
194 187: 187,
195 188: 188,
196 189: 189,
197 190: 190,
198 191: 191,
199 192: 192,
200 193: 193,
201 194: 194,
202 195: 195,
203 196: 196,
204 197: 197,
205 198: 198,
206 199: 199,
207 200: 200,
208 201: 201,
209 202: 202,
210 203: 203,
211 204: 204,
212 205: 205,
213 206: 206,
214 207: 207,
215 208: 208,
216 209: 209,
217 210: 210,
218 211: 211,
219 212: 212,
220 213: 213,
221 214: 214,
222 215: 215,
223 216: 216,
224 217: 217,
225 218: 218,
226 219: 219,
227 220: 220,
228 221: 221,
229 222: 222,
230 223: 223,
231 224: 224,
232 225: 225,
233 226: 226,
234 227: 227,
235 228: 228,
236 229: 229,
237 230: 230,
238 231: 231,
239 232: 232,
240 233: 233,
241 234: 234,
242 235: 235,
243 236: 236,
244 237: 237,
245 238: 238,
246 239: 239,
247 240: 240,
248 241: 241,
249 242: 242,
250 243: 243,
251 244: 244,
252 245: 245,
253 246: 246,
254 247: 247,
255 248: 248,
256 249: 249,
257 250: 250,
258 251: 251,
259 252: 252,
260 253: 253,
261 254: 255,
262 255: 255,
263 256: 0,
264 257: 1,
265 258: 2,
266 259: 3,
267 260: 4,
268 261: 5,
269 262: 6,
270 263: 7,
271 264: 8,
272 265: 9,
273 266: 10,
274 267: 11,
275 268: 12,
276 269: 13,
277 270: 14,
278 271: 15,
279 272: 16,
280 273: 17,
281 274: 18,
282 275: 19,
283 276: 20,
284 277: 21,
285 278: 22,
286 279: 23,
287 280: 24,
288 281: 25,
289 282: 26,
290 283: 27,
291 284: 28,
292 285: 29,
293 286: 30,
294 287: 31,
295 288: 32,
296 289: 127,
297 290: 128,
298 291: 129,
299 292: 130,
300 293: 131,
301 294: 132,
302 295: 133,
303 296: 134,
304 297: 135,
305 298: 136,
306 299: 137,
307 300: 138,
308 301: 139,
309 302: 140,
310 303: 141,
311 304: 142,
312 305: 143,
313 306: 144,
314 307: 145,
315 308: 146,
316 309: 147,
317 310: 148,
318 311: 149,
319 312: 150,
320 313: 151,
321 314: 152,
322 315: 153,
323 316: 154,
324 317: 155,
325 318: 156,
326 319: 157,
327 320: 158,
328 321: 159,
329 322: 160,
330 323: 173,
331 };