A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Allow `tokenizer_mode` and `trust_remote_code` for Detokenizer (#10211)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 11, 2024 at 02:01 UTC
5fb1f935b04c29c5c379952681a8a49ad533355d
2 files changed
+19
-5
vllm/v1/engine/llm_engine.py
+4
-1
@@ -125,7 +125,10 @@ class LLMEngine:
125
# Ping the tokenizer to ensure liveness if it runs in a
126
# different process.
127
self.tokenizer.ping()
128
- self.detokenizer = Detokenizer(self.model_config.tokenizer)
128
+ self.detokenizer = Detokenizer(
129
+ tokenizer_name=self.model_config.tokenizer,
130
+ tokenizer_mode=self.model_config.tokenizer_mode,
131
+ trust_remote_code=self.model_config.trust_remote_code)
132
133
self.generation_config_fields = _load_generation_config_dict(
134
model_config)
vllm/v1/tokenizer/detokenizer.py
+15
-4
@@ -42,13 +42,17 @@ class DetokenizerOutputs(msgspec.Struct):
42
43
class Detokenizer:
44
45
- def __init__(self, tokenizer_name: str):
45
+ def __init__(self, tokenizer_name: str, tokenizer_mode: str,
46
+ trust_remote_code: bool):
47
# FIXME(woosuk): Currently, the detokenizer is just a hacky prototype.
48
# For example, it does not terminate properly. We need to improve this.
49
self.push_port = get_open_port()
50
self.pull_port = get_open_port()
50
- self.detokenizer = DetokenizerProc(tokenizer_name, self.push_port,
51
- self.pull_port)
51
+ self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
52
+ tokenizer_mode=tokenizer_mode,
53
+ trust_remote_code=trust_remote_code,
54
+ push_port=self.push_port,
55
+ pull_port=self.pull_port)
56
self.detokenizer.start()
57
58
self.zmq_context = zmq.Context()
@@ -82,11 +86,15 @@ class DetokenizerProc(multiprocessing.Process):
86
def __init__(
87
self,
88
tokenizer_name: str,
89
+ tokenizer_mode: str,
90
+ trust_remote_code: bool,
91
pull_port: int,
92
push_port: int,
93
):
94
super().__init__()
95
self.tokenizer_name = tokenizer_name
96
+ self.tokenizer_mode = tokenizer_mode
97
+ self.trust_remote_code = trust_remote_code
98
# NOTE: The pull_port of the detokenizer should be the same as the
99
# push_port of the engine. Vice versa.
100
self.pull_port = pull_port
@@ -97,7 +105,10 @@ class DetokenizerProc(multiprocessing.Process):
105
# not picklable.
106
self.msgpack_encoder = msgpack.Encoder()
107
self.msgpack_decoder = msgpack.Decoder(DetokenizerInputs)
100
- self.tokenizer = get_tokenizer(self.tokenizer_name)
108
+ self.tokenizer = get_tokenizer(
109
+ tokenizer_name=self.tokenizer_name,
110
+ tokenizer_mode=self.tokenizer_mode,
111
+ trust_remote_code=self.trust_remote_code)
112
# req_id -> RequestState
113
self.request_states: Dict[str, RequestState] = {}
114