[Rust Frontend] Migrate to new tekken crate (#53056)
Signed-off-by: Jorge Menjivar <jorge@menjivar.ai>
Jorge Menjivar committed
Aug 31, 2026 at 19:35 UTC
225aec4809e5a5c7296a43ac6b363edfa1b06bad
2 files changed
+10
-62
rust/Cargo.lock
+9
-61
@@ -1737,12 +1737,6 @@ dependencies = [
1737
"digest",
1738
]
1739
1740
-[[package]]
1741
-name = "hound"
1742
-version = "3.5.1"
1743
-source = "registry+https://github.com/rust-lang/crates.io-index"
1744
-checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
1745
-
1740
[[package]]
1741
name = "http"
1742
version = "1.4.0"
@@ -2307,7 +2301,7 @@ dependencies = [
2301
"hf-hub",
2302
"image",
2303
"libloading",
2310
- "ndarray 0.17.2",
2304
+ "ndarray",
2305
"once_cell",
2306
"pkg-config",
2307
"rayon",
@@ -2677,21 +2671,6 @@ dependencies = [
2671
"tempfile",
2672
]
2673
2680
-[[package]]
2681
-name = "ndarray"
2682
-version = "0.16.1"
2683
-source = "registry+https://github.com/rust-lang/crates.io-index"
2684
-checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
2685
-dependencies = [
2686
- "matrixmultiply",
2687
- "num-complex",
2688
- "num-integer",
2689
- "num-traits",
2690
- "portable-atomic",
2691
- "portable-atomic-util",
2692
- "rawpointer",
2693
-]
2694
-
2674
[[package]]
2675
name = "ndarray"
2676
version = "0.17.2"
@@ -3846,18 +3825,6 @@ dependencies = [
3825
"serde_bytes",
3826
]
3827
3849
-[[package]]
3850
-name = "rubato"
3851
-version = "0.16.2"
3852
-source = "registry+https://github.com/rust-lang/crates.io-index"
3853
-checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1"
3854
-dependencies = [
3855
- "num-complex",
3856
- "num-integer",
3857
- "num-traits",
3858
- "realfft",
3859
-]
3860
-
3828
[[package]]
3829
name = "rustc-hash"
3830
version = "1.1.0"
@@ -4814,25 +4781,21 @@ dependencies = [
4781
]
4782
4783
[[package]]
4817
-name = "tekken-rs"
4818
-version = "0.1.1"
4784
+name = "tekken"
4785
+version = "0.2.0"
4786
source = "registry+https://github.com/rust-lang/crates.io-index"
4820
-checksum = "49623843103837f53f7ebe8cfafc19ccff28ff0e15e7c4b9f6ad21e36fbfde3a"
4787
+checksum = "1d1481e6bbb98c7e1b04124420b6e7c367c87fc897aae340edcdeb4e91b639d2"
4788
dependencies = [
4789
"anyhow",
4790
"base64 0.22.1",
4791
"env_logger",
4825
- "hound",
4792
"log",
4827
- "ndarray 0.16.1",
4793
"regex",
4829
- "rubato",
4794
"rustc-hash 1.1.0",
4831
- "rustfft",
4795
"serde",
4796
"serde_json",
4797
"thiserror 2.0.18",
4835
- "tiktoken-rs 0.7.0",
4798
+ "tiktoken-rs",
4799
]
4800
4801
[[package]]
@@ -4933,21 +4896,6 @@ dependencies = [
4896
"zune-jpeg",
4897
]
4898
4936
-[[package]]
4937
-name = "tiktoken-rs"
4938
-version = "0.7.0"
4939
-source = "registry+https://github.com/rust-lang/crates.io-index"
4940
-checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
4941
-dependencies = [
4942
- "anyhow",
4943
- "base64 0.22.1",
4944
- "bstr",
4945
- "fancy-regex 0.13.0",
4946
- "lazy_static",
4947
- "regex",
4948
- "rustc-hash 1.1.0",
4949
-]
4950
-
4899
[[package]]
4900
name = "tiktoken-rs"
4901
version = "0.9.1"
@@ -5779,7 +5727,7 @@ dependencies = [
5727
"serde_json",
5728
"thiserror 2.0.18",
5729
"thiserror-ext",
5782
- "tiktoken-rs 0.9.1",
5730
+ "tiktoken-rs",
5731
"tokenizers",
5732
"tokio",
5733
"tokio-stream",
@@ -5810,7 +5758,7 @@ dependencies = [
5758
"llm-multimodal",
5759
"minijinja",
5760
"minijinja-contrib",
5813
- "ndarray 0.17.2",
5761
+ "ndarray",
5762
"oss-harmony",
5763
"paste",
5764
"reqwest 0.13.4",
@@ -6095,11 +6043,11 @@ dependencies = [
6043
"rustc-hash 1.1.0",
6044
"serde",
6045
"serde_json",
6098
- "tekken-rs",
6046
+ "tekken",
6047
"tempfile",
6048
"thiserror 2.0.18",
6049
"thiserror-ext",
6102
- "tiktoken-rs 0.9.1",
6050
+ "tiktoken-rs",
6051
"tokenizers",
6052
"tokio",
6053
"tracing",
rust/Cargo.toml
+1
-1
@@ -104,7 +104,7 @@ strum = { version = "0.27.2", features = ["derive"] }
104
subenum = "1.1.3"
105
subtle = "2.6"
106
task-local = "0.1.1"
107
-tekken = { package = "tekken-rs", version = "0.1.1", default-features = false }
107
+tekken = { version = "0.2.0", default-features = false }
108
tempfile = "3.23.0"
109
thiserror = "2.0.16"
110
thiserror-ext = "0.3.0"