@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb7991aa29
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[V1] Add missing tokenizer options for `Detokenizer` (#10288)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 13, 2024 at 03:02 UTC bb7991aa291054a30f408e626273caa6769a07eb
2 files changed +15 -3
vllm/v1/engine/detokenizer.py
+9 -2
@@ -192,10 +192,17 @@ class IncrementalDetokenizer:
192
193 class Detokenizer:
194
195 - def __init__(self, tokenizer_name: str):
195 + def __init__(self,
196 + tokenizer_name: str,
197 + tokenizer_mode: str = "auto",
198 + trust_remote_code: bool = False,
199 + revision: Optional[str] = None):
200 # TODO: once we support LoRA, we should should pass the tokenizer
201 # here. We currently have two copies (this + in the LLMEngine).
198 - self.tokenizer = get_tokenizer(tokenizer_name)
202 + self.tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
203 + tokenizer_mode=tokenizer_mode,
204 + trust_remote_code=trust_remote_code,
205 + revision=revision)
206
207 # Request id -> IncrementalDetokenizer
208 self.request_states: Dict[str, IncrementalDetokenizer] = {}
vllm/v1/engine/llm_engine.py
+6 -1
@@ -53,7 +53,12 @@ class LLMEngine:
53 input_registry)
54
55 # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
56 - self.detokenizer = Detokenizer(vllm_config.model_config.tokenizer)
56 + self.detokenizer = Detokenizer(
57 + tokenizer_name=vllm_config.model_config.tokenizer,
58 + tokenizer_mode=vllm_config.model_config.tokenizer_mode,
59 + trust_remote_code=vllm_config.model_config.trust_remote_code,
60 + revision=vllm_config.model_config.tokenizer_revision,
61 + )
62
63 # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
64 self.engine_core = EngineCoreClient.make_client(