A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Pass `tokenizer_revision` when getting tokenizer in openai serving (#4214)
Chirag Jain committed
Apr 20, 2024 at 05:43 UTC
bc9df1571b8002738eb8db70a07f552e32feb75f
1 file changed
+1
vllm/entrypoints/openai/serving_engine.py
+1
@@ -68,6 +68,7 @@ class OpenAIServing:
68
self.tokenizer = get_tokenizer(
69
engine_model_config.tokenizer,
70
tokenizer_mode=engine_model_config.tokenizer_mode,
71
+ tokenizer_revision=engine_model_config.tokenizer_revision,
72
trust_remote_code=engine_model_config.trust_remote_code,
73
truncation_side="left")
74