The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | #include <cstdint> |
| 2 | #include <cstdio> |
| 3 | #include <random> |
| 4 | #include <string> |
| 5 | |
| 6 | #include "Utf16ToUtf8.hpp" |
| 7 | |
| 8 | using margelo::nitro::t3highlights::appendUtf16AsUtf8; |
| 9 | |
| 10 | static int failures = 0; |
| 11 | |
| 12 | static std::string encode(const std::u16string& source, std::string out = {}) { |
| 13 | appendUtf16AsUtf8(out, source.data(), source.size()); |
| 14 | return out; |
| 15 | } |
| 16 | |
| 17 | static void expectBytes(const char* name, const std::string& actual, const std::string& expected) { |
| 18 | if (actual == expected) return; |
| 19 | failures++; |
| 20 | std::fprintf(stderr, "%s: expected", name); |
| 21 | for (unsigned char byte : expected) std::fprintf(stderr, " %02x", byte); |
| 22 | std::fprintf(stderr, ", got"); |
| 23 | for (unsigned char byte : actual) std::fprintf(stderr, " %02x", byte); |
| 24 | std::fprintf(stderr, "\n"); |
| 25 | } |
| 26 | |
| 27 | // One code unit or valid pair at a time, as a slow reference. |
| 28 | static std::string reference(const std::u16string& source) { |
| 29 | std::string out; |
| 30 | for (size_t at = 0; at < source.size(); at++) { |
| 31 | uint32_t point = source[at]; |
| 32 | if (point >= 0xd800 && point <= 0xdbff && at + 1 < source.size() && source[at + 1] >= 0xdc00 && |
| 33 | source[at + 1] <= 0xdfff) { |
| 34 | point = 0x10000 + ((point - 0xd800) << 10) + (source[++at] - 0xdc00); |
| 35 | } else if (point >= 0xd800 && point <= 0xdfff) { |
| 36 | point = 0xfffd; |
| 37 | } |
| 38 | if (point < 0x80) { |
| 39 | out += static_cast<char>(point); |
| 40 | } else if (point < 0x800) { |
| 41 | out += static_cast<char>(0xc0 | (point >> 6)); |
| 42 | out += static_cast<char>(0x80 | (point & 0x3f)); |
| 43 | } else if (point < 0x10000) { |
| 44 | out += static_cast<char>(0xe0 | (point >> 12)); |
| 45 | out += static_cast<char>(0x80 | ((point >> 6) & 0x3f)); |
| 46 | out += static_cast<char>(0x80 | (point & 0x3f)); |
| 47 | } else { |
| 48 | out += static_cast<char>(0xf0 | (point >> 18)); |
| 49 | out += static_cast<char>(0x80 | ((point >> 12) & 0x3f)); |
| 50 | out += static_cast<char>(0x80 | ((point >> 6) & 0x3f)); |
| 51 | out += static_cast<char>(0x80 | (point & 0x3f)); |
| 52 | } |
| 53 | } |
| 54 | return out; |
| 55 | } |
| 56 | |
| 57 | int main() { |
| 58 | expectBytes("empty", encode(u""), ""); |
| 59 | expectBytes("nul", encode(std::u16string(u"a\0b", 3)), std::string("a\0b", 3)); |
| 60 | expectBytes("two-byte", encode(u"\u00e9\u07ff"), "\xc3\xa9\xdf\xbf"); |
| 61 | expectBytes("three-byte", encode(u"\u0800\u65e5\uffff"), "\xe0\xa0\x80\xe6\x97\xa5\xef\xbf\xbf"); |
| 62 | expectBytes("pair", encode(u"\U0001F389"), "\xf0\x9f\x8e\x89"); |
| 63 | expectBytes("lone high at end", encode(u"a\xd83c"), "a\xef\xbf\xbd"); |
| 64 | expectBytes("lone high before text", encode(u"\xd83cz"), "\xef\xbf\xbdz"); |
| 65 | expectBytes("high high low", encode(u"\xd83c\xd83c\xdf89"), "\xef\xbf\xbd\xf0\x9f\x8e\x89"); |
| 66 | expectBytes("lone low", encode(u"\xdf89"), "\xef\xbf\xbd"); |
| 67 | expectBytes("appends", encode(u"\u00e9", "ok "), "ok \xc3\xa9"); |
| 68 | // A high surrogate at the end of the input must not pair with the unit after it. |
| 69 | const std::u16string pair = u"\U0001F389"; |
| 70 | std::string split; |
| 71 | appendUtf16AsUtf8(split, pair.data(), 1); |
| 72 | expectBytes("high at input end", split, "\xef\xbf\xbd"); |
| 73 | |
| 74 | // Runs either side of the four-unit ASCII fast path, with a non-ASCII unit at every offset. |
| 75 | for (size_t length = 0; length <= 12; length++) { |
| 76 | for (size_t at = 0; at <= length; at++) { |
| 77 | std::u16string source(length, u'x'); |
| 78 | if (at < length) source[at] = 0x00e9; |
| 79 | expectBytes("fast path boundary", encode(source), reference(source)); |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | const char16_t pool[] = {u'a', u'\n', 0x7f, 0x80, 0x7ff, 0x800, 0xd7ff, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xe000, 0xffff}; |
| 84 | std::mt19937 random(7); |
| 85 | for (int run = 0; run < 20000; run++) { |
| 86 | std::u16string source(random() % 24, u' '); |
| 87 | for (auto& unit : source) unit = pool[random() % (sizeof(pool) / sizeof(pool[0]))]; |
| 88 | expectBytes("random", encode(source), reference(source)); |
| 89 | } |
| 90 | |
| 91 | if (failures > 0) return 1; |
| 92 | std::printf("utf16-to-utf8: ok\n"); |
| 93 | return 0; |
| 94 | } |