The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | #include "t3_highlights.h" |
| 2 | |
| 3 | #include <stdbool.h> |
| 4 | #include <string.h> |
| 5 | |
| 6 | #include "highlights.h" |
| 7 | #include "wasm-rt-impl.h" |
| 8 | |
| 9 | // The control block and buffer layout mirror the JavaScript glue in |
| 10 | // @pierre/highlights (`HighlightsHighlighter` in dist/highlighter.js). |
| 11 | // INPUT_OFFSET is its `inputPtr`: the lexers' static data ends there. |
| 12 | #define INPUT_OFFSET 81920u |
| 13 | #define INPUT_PADDING 96u |
| 14 | #define MODE_UTF16_LINE_RECORDS 3u |
| 15 | // Like the JavaScript glue, a small input after a large one gets a fresh |
| 16 | // instance so the grown memory is released. |
| 17 | #define RESET_AFTER_PAGES 128u |
| 18 | |
| 19 | struct w2c_env { |
| 20 | w2c_highlights* instance; |
| 21 | }; |
| 22 | |
| 23 | static w2c_highlights instance; |
| 24 | static struct w2c_env env = {&instance}; |
| 25 | static bool instantiated = false; |
| 26 | |
| 27 | #include "generated/id_ranges.inc" |
| 28 | |
| 29 | static bool in_ranges(const uint32_t ranges[][2], size_t count, uint32_t code_point) { |
| 30 | size_t low = 0; |
| 31 | size_t high = count; |
| 32 | while (low < high) { |
| 33 | size_t mid = low + (high - low) / 2; |
| 34 | if (code_point < ranges[mid][0]) { |
| 35 | high = mid; |
| 36 | } else if (code_point > ranges[mid][1]) { |
| 37 | low = mid + 1; |
| 38 | } else { |
| 39 | return true; |
| 40 | } |
| 41 | } |
| 42 | return false; |
| 43 | } |
| 44 | |
| 45 | // Decodes like the JavaScript glue, including its leniency for malformed |
| 46 | // sequences, so both builds classify the same bytes the same way. |
| 47 | static u32 id_class(struct w2c_env* host, u32 ptr, u32 len, bool start) { |
| 48 | wasm_rt_memory_t* memory = w2c_highlights_memory(host->instance); |
| 49 | if ((uint64_t)ptr + 4 > memory->size) return 0; |
| 50 | const uint8_t* b = memory->data + ptr; |
| 51 | uint32_t code_point; |
| 52 | if (len == 2) { |
| 53 | code_point = (uint32_t)(b[0] & 31) << 6 | (b[1] & 63); |
| 54 | } else if (len == 3) { |
| 55 | code_point = (uint32_t)(b[0] & 15) << 12 | (uint32_t)(b[1] & 63) << 6 | (b[2] & 63); |
| 56 | } else { |
| 57 | code_point = (uint32_t)(b[0] & 7) << 18 | (uint32_t)(b[1] & 63) << 12 | |
| 58 | (uint32_t)(b[2] & 63) << 6 | (b[3] & 63); |
| 59 | } |
| 60 | if (code_point > 0x10ffff) return 0; |
| 61 | return start ? in_ranges(id_start_ranges, sizeof(id_start_ranges) / sizeof(id_start_ranges[0]), |
| 62 | code_point) |
| 63 | : in_ranges(id_continue_ranges, |
| 64 | sizeof(id_continue_ranges) / sizeof(id_continue_ranges[0]), code_point); |
| 65 | } |
| 66 | |
| 67 | u32 w2c_env_is_id_start(struct w2c_env* host, u32 ptr, u32 len) { |
| 68 | return id_class(host, ptr, len, true); |
| 69 | } |
| 70 | |
| 71 | u32 w2c_env_is_id_continue(struct w2c_env* host, u32 ptr, u32 len) { |
| 72 | return id_class(host, ptr, len, false); |
| 73 | } |
| 74 | |
| 75 | static void reset_instance(void) { |
| 76 | if (!instantiated) return; |
| 77 | wasm2c_highlights_free(&instance); |
| 78 | instantiated = false; |
| 79 | } |
| 80 | |
| 81 | static void ensure_instance(void) { |
| 82 | if (instantiated) return; |
| 83 | wasm_rt_init(); |
| 84 | wasm2c_highlights_instantiate(&instance, &env); |
| 85 | instantiated = true; |
| 86 | } |
| 87 | |
| 88 | static void write_u32(uint8_t* at, uint32_t value) { |
| 89 | at[0] = value & 0xff; |
| 90 | at[1] = (value >> 8) & 0xff; |
| 91 | at[2] = (value >> 16) & 0xff; |
| 92 | at[3] = value >> 24; |
| 93 | } |
| 94 | |
| 95 | static uint32_t read_u32(const uint8_t* at) { |
| 96 | return (uint32_t)at[0] | (uint32_t)at[1] << 8 | (uint32_t)at[2] << 16 | (uint32_t)at[3] << 24; |
| 97 | } |
| 98 | |
| 99 | int64_t t3_highlights_tokenize(const uint8_t* source, size_t length, uint8_t language, |
| 100 | const uint32_t** records) { |
| 101 | *records = NULL; |
| 102 | if (length > UINT32_MAX - WASM_DEFAULT_PAGE_SIZE * 2) return -1; |
| 103 | |
| 104 | if (instantiated && w2c_highlights_memory(&instance)->pages > RESET_AFTER_PAGES && |
| 105 | length <= WASM_DEFAULT_PAGE_SIZE) { |
| 106 | reset_instance(); |
| 107 | } |
| 108 | ensure_instance(); |
| 109 | |
| 110 | wasm_rt_memory_t* memory = w2c_highlights_memory(&instance); |
| 111 | uint64_t needed_pages = |
| 112 | ((uint64_t)INPUT_OFFSET + length + INPUT_PADDING + WASM_DEFAULT_PAGE_SIZE - 1) / |
| 113 | WASM_DEFAULT_PAGE_SIZE; |
| 114 | if (needed_pages > memory->pages && |
| 115 | wasm_rt_grow_memory(memory, needed_pages - memory->pages) == (uint64_t)-1) { |
| 116 | return -1; |
| 117 | } |
| 118 | |
| 119 | uint8_t* data = memory->data; |
| 120 | if (length > 0) memcpy(data + INPUT_OFFSET, source, length); |
| 121 | data[INPUT_OFFSET + length] = 0; |
| 122 | data[0] = language; |
| 123 | data[1] = MODE_UTF16_LINE_RECORDS; |
| 124 | write_u32(data + 2, (uint32_t)length); |
| 125 | |
| 126 | // `wasm_rt_impl_try` without the exception-handling runtime this module does |
| 127 | // not use. A trap longjmps back here. |
| 128 | WASM_RT_SAVE_STACK_DEPTH(); |
| 129 | if (WASM_RT_SETJMP(g_wasm_rt_jmp_buf) != WASM_RT_TRAP_NONE) { |
| 130 | reset_instance(); |
| 131 | return -1; |
| 132 | } |
| 133 | w2c_highlights_highlight(&instance); |
| 134 | |
| 135 | // The lexer can grow memory, which moves it. |
| 136 | data = memory->data; |
| 137 | uint32_t start = read_u32(data + 6); |
| 138 | uint32_t byte_length = read_u32(data + 10); |
| 139 | if ((uint64_t)start + byte_length > memory->size || start % 4 != 0) return -1; |
| 140 | *records = (const uint32_t*)(data + start); |
| 141 | return byte_length / 4; |
| 142 | } |