@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5fb1f935b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Allow `tokenizer_mode` and `trust_remote_code` for Detokenizer (#10211)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 11, 2024 at 02:01 UTC 5fb1f935b04c29c5c379952681a8a49ad533355d
2 files changed +19 -5
vllm/v1/engine/llm_engine.py
+4 -1
@@ -125,7 +125,10 @@ class LLMEngine:
125 # Ping the tokenizer to ensure liveness if it runs in a
126 # different process.
127 self.tokenizer.ping()
128 - self.detokenizer = Detokenizer(self.model_config.tokenizer)
128 + self.detokenizer = Detokenizer(
129 + tokenizer_name=self.model_config.tokenizer,
130 + tokenizer_mode=self.model_config.tokenizer_mode,
131 + trust_remote_code=self.model_config.trust_remote_code)
132
133 self.generation_config_fields = _load_generation_config_dict(
134 model_config)
vllm/v1/tokenizer/detokenizer.py
+15 -4
@@ -42,13 +42,17 @@ class DetokenizerOutputs(msgspec.Struct):
42
43 class Detokenizer:
44
45 - def __init__(self, tokenizer_name: str):
45 + def __init__(self, tokenizer_name: str, tokenizer_mode: str,
46 + trust_remote_code: bool):
47 # FIXME(woosuk): Currently, the detokenizer is just a hacky prototype.
48 # For example, it does not terminate properly. We need to improve this.
49 self.push_port = get_open_port()
50 self.pull_port = get_open_port()
50 - self.detokenizer = DetokenizerProc(tokenizer_name, self.push_port,
51 - self.pull_port)
51 + self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
52 + tokenizer_mode=tokenizer_mode,
53 + trust_remote_code=trust_remote_code,
54 + push_port=self.push_port,
55 + pull_port=self.pull_port)
56 self.detokenizer.start()
57
58 self.zmq_context = zmq.Context()
@@ -82,11 +86,15 @@ class DetokenizerProc(multiprocessing.Process):
86 def __init__(
87 self,
88 tokenizer_name: str,
89 + tokenizer_mode: str,
90 + trust_remote_code: bool,
91 pull_port: int,
92 push_port: int,
93 ):
94 super().__init__()
95 self.tokenizer_name = tokenizer_name
96 + self.tokenizer_mode = tokenizer_mode
97 + self.trust_remote_code = trust_remote_code
98 # NOTE: The pull_port of the detokenizer should be the same as the
99 # push_port of the engine. Vice versa.
100 self.pull_port = pull_port
@@ -97,7 +105,10 @@ class DetokenizerProc(multiprocessing.Process):
105 # not picklable.
106 self.msgpack_encoder = msgpack.Encoder()
107 self.msgpack_decoder = msgpack.Decoder(DetokenizerInputs)
100 - self.tokenizer = get_tokenizer(self.tokenizer_name)
108 + self.tokenizer = get_tokenizer(
109 + tokenizer_name=self.tokenizer_name,
110 + tokenizer_mode=self.tokenizer_mode,
111 + trust_remote_code=self.trust_remote_code)
112 # req_id -> RequestState
113 self.request_states: Dict[str, RequestState] = {}
114