The best way to control Claude Code, Codex and more from any computer or phone

main
cpp 94 lines 3.7 KB
Raw
1 #include <cstdint>
2 #include <cstdio>
3 #include <random>
4 #include <string>
5
6 #include "Utf16ToUtf8.hpp"
7
8 using margelo::nitro::t3highlights::appendUtf16AsUtf8;
9
10 static int failures = 0;
11
12 static std::string encode(const std::u16string& source, std::string out = {}) {
13 appendUtf16AsUtf8(out, source.data(), source.size());
14 return out;
15 }
16
17 static void expectBytes(const char* name, const std::string& actual, const std::string& expected) {
18 if (actual == expected) return;
19 failures++;
20 std::fprintf(stderr, "%s: expected", name);
21 for (unsigned char byte : expected) std::fprintf(stderr, " %02x", byte);
22 std::fprintf(stderr, ", got");
23 for (unsigned char byte : actual) std::fprintf(stderr, " %02x", byte);
24 std::fprintf(stderr, "\n");
25 }
26
27 // One code unit or valid pair at a time, as a slow reference.
28 static std::string reference(const std::u16string& source) {
29 std::string out;
30 for (size_t at = 0; at < source.size(); at++) {
31 uint32_t point = source[at];
32 if (point >= 0xd800 && point <= 0xdbff && at + 1 < source.size() && source[at + 1] >= 0xdc00 &&
33 source[at + 1] <= 0xdfff) {
34 point = 0x10000 + ((point - 0xd800) << 10) + (source[++at] - 0xdc00);
35 } else if (point >= 0xd800 && point <= 0xdfff) {
36 point = 0xfffd;
37 }
38 if (point < 0x80) {
39 out += static_cast<char>(point);
40 } else if (point < 0x800) {
41 out += static_cast<char>(0xc0 | (point >> 6));
42 out += static_cast<char>(0x80 | (point & 0x3f));
43 } else if (point < 0x10000) {
44 out += static_cast<char>(0xe0 | (point >> 12));
45 out += static_cast<char>(0x80 | ((point >> 6) & 0x3f));
46 out += static_cast<char>(0x80 | (point & 0x3f));
47 } else {
48 out += static_cast<char>(0xf0 | (point >> 18));
49 out += static_cast<char>(0x80 | ((point >> 12) & 0x3f));
50 out += static_cast<char>(0x80 | ((point >> 6) & 0x3f));
51 out += static_cast<char>(0x80 | (point & 0x3f));
52 }
53 }
54 return out;
55 }
56
57 int main() {
58 expectBytes("empty", encode(u""), "");
59 expectBytes("nul", encode(std::u16string(u"a\0b", 3)), std::string("a\0b", 3));
60 expectBytes("two-byte", encode(u"\u00e9\u07ff"), "\xc3\xa9\xdf\xbf");
61 expectBytes("three-byte", encode(u"\u0800\u65e5\uffff"), "\xe0\xa0\x80\xe6\x97\xa5\xef\xbf\xbf");
62 expectBytes("pair", encode(u"\U0001F389"), "\xf0\x9f\x8e\x89");
63 expectBytes("lone high at end", encode(u"a\xd83c"), "a\xef\xbf\xbd");
64 expectBytes("lone high before text", encode(u"\xd83cz"), "\xef\xbf\xbdz");
65 expectBytes("high high low", encode(u"\xd83c\xd83c\xdf89"), "\xef\xbf\xbd\xf0\x9f\x8e\x89");
66 expectBytes("lone low", encode(u"\xdf89"), "\xef\xbf\xbd");
67 expectBytes("appends", encode(u"\u00e9", "ok "), "ok \xc3\xa9");
68 // A high surrogate at the end of the input must not pair with the unit after it.
69 const std::u16string pair = u"\U0001F389";
70 std::string split;
71 appendUtf16AsUtf8(split, pair.data(), 1);
72 expectBytes("high at input end", split, "\xef\xbf\xbd");
73
74 // Runs either side of the four-unit ASCII fast path, with a non-ASCII unit at every offset.
75 for (size_t length = 0; length <= 12; length++) {
76 for (size_t at = 0; at <= length; at++) {
77 std::u16string source(length, u'x');
78 if (at < length) source[at] = 0x00e9;
79 expectBytes("fast path boundary", encode(source), reference(source));
80 }
81 }
82
83 const char16_t pool[] = {u'a', u'\n', 0x7f, 0x80, 0x7ff, 0x800, 0xd7ff, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xe000, 0xffff};
84 std::mt19937 random(7);
85 for (int run = 0; run < 20000; run++) {
86 std::u16string source(random() % 24, u' ');
87 for (auto& unit : source) unit = pool[random() % (sizeof(pool) / sizeof(pool[0]))];
88 expectBytes("random", encode(source), reference(source));
89 }
90
91 if (failures > 0) return 1;
92 std::printf("utf16-to-utf8: ok\n");
93 return 0;
94 }