A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Fix Detokenizer loading in `AsyncLLM` (#10997)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Dec 9, 2024 at 08:27 UTC
c690357928fd2812f450bfb0c3629a816f5e9a55
1 file changed
+6
-1
vllm/v1/engine/async_llm.py
+6
-1
@@ -65,7 +65,12 @@ class AsyncLLM(EngineClient):
65
input_registry)
66
67
# Detokenizer (converts EngineCoreOutputs --> RequestOutput).
68
- self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer)
68
+ self.detokenizer = Detokenizer(
69
+ tokenizer_name=vllm_config.model_config.tokenizer,
70
+ tokenizer_mode=vllm_config.model_config.tokenizer_mode,
71
+ trust_remote_code=vllm_config.model_config.trust_remote_code,
72
+ revision=vllm_config.model_config.tokenizer_revision,
73
+ )
74
75
# EngineCore (starts the engine in background process).
76
self.engine_core = EngineCoreClient.make_client(