A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
Intialize io_thread_pool attribute in the beginning. (#18331)
Signed-off-by: rabi <ramishra@redhat.com>
Rabi Mishra committed
May 22, 2025 at 08:51 UTC
acb54ca8e11a72592de963cf2f7477635349d526
1 file changed
+1
-1
vllm/v1/executor/multiproc_executor.py
+1
-1
@@ -50,6 +50,7 @@ class MultiprocExecutor(Executor):
50
self.is_failed = False
51
self.shutdown_event = threading.Event()
52
self.failure_callback: Optional[FailureCallback] = None
53
+ self.io_thread_pool: Optional[ThreadPoolExecutor] = None
54
55
self.world_size = self.parallel_config.world_size
56
tensor_parallel_size = self.parallel_config.tensor_parallel_size
@@ -107,7 +108,6 @@ class MultiprocExecutor(Executor):
108
109
# For pipeline parallel, we use a thread pool for asynchronous
110
# execute_model.
110
- self.io_thread_pool: Optional[ThreadPoolExecutor] = None
111
if self.max_concurrent_batches > 1:
112
# Note: must use only 1 IO thread to keep dequeue sequence
113
# from the response queue