A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Use slow tokenizer for open llama models (#168)
Woosuk Kwon committed
Jun 19, 2023 at 23:19 UTC
67d96c29fba9b72cb4c4edbc26211c208a00ebdd
1 file changed
+6
-1
vllm/engine/tokenizer_utils.py
+6
-1
@@ -17,7 +17,12 @@ def get_tokenizer(
17
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18
"""Gets a tokenizer for the given model name via Huggingface."""
19
config = AutoConfig.from_pretrained(model_name)
20
- if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
20
+ if "open_llama" in model_name:
21
+ kwargs["use_fast"] = False
22
+ logger.info(
23
+ "OpenLLaMA models do not support the fast tokenizer. "
24
+ "Using the slow tokenizer instead.")
25
+ elif config.model_type == "llama" and getattr(kwargs, "use_fast", True):
26
# LLaMA fast tokenizer causes protobuf errors in some environments.
27
# However, we found that the below LLaMA fast tokenizer works well in
28
# most environments.