| 1 | [package] |
| 2 | name = "vllm-tokenizer" |
| 3 | version.workspace = true |
| 4 | edition.workspace = true |
| 5 | license.workspace = true |
| 6 | |
| 7 | [features] |
| 8 | test-utils = [] |
| 9 | |
| 10 | [dependencies] |
| 11 | base64.workspace = true |
| 12 | fastokens.workspace = true |
| 13 | riptoken.workspace = true |
| 14 | rustc-hash.workspace = true |
| 15 | serde.workspace = true |
| 16 | serde_json.workspace = true |
| 17 | smallvec.workspace = true |
| 18 | tekken.workspace = true |
| 19 | thiserror.workspace = true |
| 20 | thiserror-ext.workspace = true |
| 21 | tiktoken-rs.workspace = true |
| 22 | tokenizers.workspace = true |
| 23 | tracing.workspace = true |
| 24 | |
| 25 | [dev-dependencies] |
| 26 | criterion.workspace = true |
| 27 | hf-hub.workspace = true |
| 28 | reqwest.workspace = true |
| 29 | tempfile.workspace = true |
| 30 | tokio.workspace = true |
| 31 | |
| 32 | [[bench]] |
| 33 | name = "hf" |
| 34 | harness = false |
| 35 | |
| 36 | [[bench]] |
| 37 | name = "tiktoken" |
| 38 | harness = false |
| 39 | |
| 40 | [lints] |
| 41 | workspace = true |