@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67d96c29fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Use slow tokenizer for open llama models (#168)

Woosuk Kwon committed Jun 19, 2023 at 23:19 UTC 67d96c29fba9b72cb4c4edbc26211c208a00ebdd
1 file changed +6 -1
vllm/engine/tokenizer_utils.py
+6 -1
@@ -17,7 +17,12 @@ def get_tokenizer(
17 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18 """Gets a tokenizer for the given model name via Huggingface."""
19 config = AutoConfig.from_pretrained(model_name)
20 - if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
20 + if "open_llama" in model_name:
21 + kwargs["use_fast"] = False
22 + logger.info(
23 + "OpenLLaMA models do not support the fast tokenizer. "
24 + "Using the slow tokenizer instead.")
25 + elif config.model_type == "llama" and getattr(kwargs, "use_fast", True):
26 # LLaMA fast tokenizer causes protobuf errors in some environments.
27 # However, we found that the below LLaMA fast tokenizer works well in
28 # most environments.