A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Enable LLaMA fast tokenizer (#132)
Woosuk Kwon committed
May 28, 2023 at 02:51 UTC
337871c6fd581b74949849ad645064318896801b
2 files changed
+11
-7
cacheflow/sampling_params.py
+1
-1
@@ -129,7 +129,7 @@ class SamplingParams:
129
f"frequency_penalty={self.frequency_penalty}, "
130
f"temperature={self.temperature}, "
131
f"top_p={self.top_p}, "
132
- f"top_k={self.top_k},"
132
+ f"top_k={self.top_k}, "
133
f"use_beam_search={self.use_beam_search}, "
134
f"stop={self.stop}, "
135
f"ignore_eos={self.ignore_eos}, "
cacheflow/server/tokenizer_utils.py
+10
-6
@@ -7,11 +7,7 @@ from cacheflow.logger import init_logger
7
8
logger = init_logger(__name__)
9
10
-_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
11
- # LLaMA fast tokenizer has a bug related to protobuf.
12
- # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
13
- "llama",
14
-]
10
+_MODEL_TYPES_WITH_SLOW_TOKENIZER = []
11
12
13
def get_tokenizer(
@@ -20,7 +16,15 @@ def get_tokenizer(
16
**kwargs,
17
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18
config = AutoConfig.from_pretrained(model_name)
23
- if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
19
+ if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
20
+ # LLaMA fast tokenizer causes protobuf errors in some environments.
21
+ # However, we found that the below LLaMA fast tokenizer works well in
22
+ # most environments.
23
+ model_name = "hf-internal-testing/llama-tokenizer"
24
+ logger.info(
25
+ f"Using the LLaMA fast tokenizer in '{model_name}' to avoid "
26
+ "potential protobuf errors.")
27
+ elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
28
if getattr(kwargs, "use_fast", False) == True:
29
raise ValueError(
30
f"Cannot use the fast tokenizer for {config.model_type} due to "