A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Fix detokenizer ports (#10224)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 11, 2024 at 10:42 UTC
f9dadfbee331aeff9cb45c94e635ab8e16335a10
1 file changed
+6
-4
vllm/v1/tokenizer/detokenizer.py
+6
-4
@@ -48,11 +48,13 @@ class Detokenizer:
48
# For example, it does not terminate properly. We need to improve this.
49
self.push_port = get_open_port()
50
self.pull_port = get_open_port()
51
+ # NOTE: The push port of the engine process should be the same as the
52
+ # pull port of the detokenizer process. Vice versa.
53
self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
54
tokenizer_mode=tokenizer_mode,
55
trust_remote_code=trust_remote_code,
54
- push_port=self.push_port,
55
- pull_port=self.pull_port)
56
+ push_port=self.pull_port,
57
+ pull_port=self.push_port)
58
self.detokenizer.start()
59
60
self.zmq_context = zmq.Context()
@@ -95,8 +97,8 @@ class DetokenizerProc(multiprocessing.Process):
97
self.tokenizer_name = tokenizer_name
98
self.tokenizer_mode = tokenizer_mode
99
self.trust_remote_code = trust_remote_code
98
- # NOTE: The pull_port of the detokenizer should be the same as the
99
- # push_port of the engine. Vice versa.
100
+ # NOTE: The pull_port of the detokenizer process should be the same as
101
+ # the push_port of the engine process. Vice versa.
102
self.pull_port = pull_port
103
self.push_port = push_port
104