@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c690357928
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Fix Detokenizer loading in `AsyncLLM` (#10997)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Dec 9, 2024 at 08:27 UTC c690357928fd2812f450bfb0c3629a816f5e9a55
1 file changed +6 -1
vllm/v1/engine/async_llm.py
+6 -1
@@ -65,7 +65,12 @@ class AsyncLLM(EngineClient):
65 input_registry)
66
67 # Detokenizer (converts EngineCoreOutputs --> RequestOutput).
68 - self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer)
68 + self.detokenizer = Detokenizer(
69 + tokenizer_name=vllm_config.model_config.tokenizer,
70 + tokenizer_mode=vllm_config.model_config.tokenizer_mode,
71 + trust_remote_code=vllm_config.model_config.trust_remote_code,
72 + revision=vllm_config.model_config.tokenizer_revision,
73 + )
74
75 # EngineCore (starts the engine in background process).
76 self.engine_core = EngineCoreClient.make_client(