A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[BugFix] Apply get_cached_tokenizer to the tokenizer setter of LLM (#5207)
Co-authored-by: qiujiawei9 <qiujiawei9@jd.com>
DriverSong committed
Jun 6, 2024 at 01:59 UTC
eb8fcd266686570a1cfdcda4af73af0e27b0f3d8
1 file changed
+9
-1
vllm/entrypoints/llm.py
+9
-1
@@ -14,6 +14,7 @@ from vllm.lora.request import LoRARequest
14
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15
from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams
17
+from vllm.transformers_utils.tokenizer import get_cached_tokenizer
18
from vllm.usage.usage_lib import UsageContext
19
from vllm.utils import Counter, deprecate_kwargs
20
@@ -152,7 +153,14 @@ class LLM:
153
self,
154
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
155
) -> None:
155
- self.llm_engine.tokenizer.tokenizer = tokenizer
156
+ # While CachedTokenizer is dynamic, have no choice but
157
+ # compare class name. Misjudgment will arise from
158
+ # user-defined tokenizer started with 'Cached'
159
+ if tokenizer.__class__.__name__.startswith("Cached"):
160
+ self.llm_engine.tokenizer.tokenizer = tokenizer
161
+ else:
162
+ self.llm_engine.tokenizer.tokenizer = get_cached_tokenizer(
163
+ tokenizer)
164
165
@overload # LEGACY: single (prompt + optional token ids)
166
def generate(