@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb8fcd2666
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[BugFix] Apply get_cached_tokenizer to the tokenizer setter of LLM (#5207)

Co-authored-by: qiujiawei9 <qiujiawei9@jd.com>

DriverSong committed Jun 6, 2024 at 01:59 UTC eb8fcd266686570a1cfdcda4af73af0e27b0f3d8
1 file changed +9 -1
vllm/entrypoints/llm.py
+9 -1
@@ -14,6 +14,7 @@ from vllm.lora.request import LoRARequest
14 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15 from vllm.pooling_params import PoolingParams
16 from vllm.sampling_params import SamplingParams
17 +from vllm.transformers_utils.tokenizer import get_cached_tokenizer
18 from vllm.usage.usage_lib import UsageContext
19 from vllm.utils import Counter, deprecate_kwargs
20
@@ -152,7 +153,14 @@ class LLM:
153 self,
154 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
155 ) -> None:
155 - self.llm_engine.tokenizer.tokenizer = tokenizer
156 + # While CachedTokenizer is dynamic, have no choice but
157 + # compare class name. Misjudgment will arise from
158 + # user-defined tokenizer started with 'Cached'
159 + if tokenizer.__class__.__name__.startswith("Cached"):
160 + self.llm_engine.tokenizer.tokenizer = tokenizer
161 + else:
162 + self.llm_engine.tokenizer.tokenizer = get_cached_tokenizer(
163 + tokenizer)
164
165 @overload # LEGACY: single (prompt + optional token ids)
166 def generate(