@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f9dadfbee3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Fix detokenizer ports (#10224)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 11, 2024 at 10:42 UTC f9dadfbee331aeff9cb45c94e635ab8e16335a10
1 file changed +6 -4
vllm/v1/tokenizer/detokenizer.py
+6 -4
@@ -48,11 +48,13 @@ class Detokenizer:
48 # For example, it does not terminate properly. We need to improve this.
49 self.push_port = get_open_port()
50 self.pull_port = get_open_port()
51 + # NOTE: The push port of the engine process should be the same as the
52 + # pull port of the detokenizer process. Vice versa.
53 self.detokenizer = DetokenizerProc(tokenizer_name=tokenizer_name,
54 tokenizer_mode=tokenizer_mode,
55 trust_remote_code=trust_remote_code,
54 - push_port=self.push_port,
55 - pull_port=self.pull_port)
56 + push_port=self.pull_port,
57 + pull_port=self.push_port)
58 self.detokenizer.start()
59
60 self.zmq_context = zmq.Context()
@@ -95,8 +97,8 @@ class DetokenizerProc(multiprocessing.Process):
97 self.tokenizer_name = tokenizer_name
98 self.tokenizer_mode = tokenizer_mode
99 self.trust_remote_code = trust_remote_code
98 - # NOTE: The pull_port of the detokenizer should be the same as the
99 - # push_port of the engine. Vice versa.
100 + # NOTE: The pull_port of the detokenizer process should be the same as
101 + # the push_port of the engine process. Vice versa.
102 self.pull_port = pull_port
103 self.push_port = push_port
104