@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 337871c6fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Enable LLaMA fast tokenizer (#132)

Woosuk Kwon committed May 28, 2023 at 02:51 UTC 337871c6fd581b74949849ad645064318896801b
2 files changed +11 -7
cacheflow/sampling_params.py
+1 -1
@@ -129,7 +129,7 @@ class SamplingParams:
129 f"frequency_penalty={self.frequency_penalty}, "
130 f"temperature={self.temperature}, "
131 f"top_p={self.top_p}, "
132 - f"top_k={self.top_k},"
132 + f"top_k={self.top_k}, "
133 f"use_beam_search={self.use_beam_search}, "
134 f"stop={self.stop}, "
135 f"ignore_eos={self.ignore_eos}, "
cacheflow/server/tokenizer_utils.py
+10 -6
@@ -7,11 +7,7 @@ from cacheflow.logger import init_logger
7
8 logger = init_logger(__name__)
9
10 -_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
11 - # LLaMA fast tokenizer has a bug related to protobuf.
12 - # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
13 - "llama",
14 -]
10 +_MODEL_TYPES_WITH_SLOW_TOKENIZER = []
11
12
13 def get_tokenizer(
@@ -20,7 +16,15 @@ def get_tokenizer(
16 **kwargs,
17 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18 config = AutoConfig.from_pretrained(model_name)
23 - if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
19 + if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
20 + # LLaMA fast tokenizer causes protobuf errors in some environments.
21 + # However, we found that the below LLaMA fast tokenizer works well in
22 + # most environments.
23 + model_name = "hf-internal-testing/llama-tokenizer"
24 + logger.info(
25 + f"Using the LLaMA fast tokenizer in '{model_name}' to avoid "
26 + "potential protobuf errors.")
27 + elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
28 if getattr(kwargs, "use_fast", False) == True:
29 raise ValueError(
30 f"Cannot use the fast tokenizer for {config.model_type} due to "