@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ed646b7aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Distributed][Core] Support Py39 and Py38 for PP (#6120)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

Murali Andoorveedu committed Jul 3, 2024 at 17:52 UTC 0ed646b7aa3b434b2fcb6f6b6e725570879cb89e
2 files changed +10 -6
vllm/executor/executor_base.py
+1 -6
@@ -123,12 +123,7 @@ class ExecutorAsyncBase(ExecutorBase):
123 multimodal_config: Optional[MultiModalConfig],
124 speculative_config: Optional[SpeculativeConfig],
125 ) -> None:
126 - # This locks each pipeline parallel stage so multiple virtual engines
127 - # can't execute on the same stage at the same time
128 - self.pp_locks = [
129 - asyncio.Lock()
130 - for _ in range(parallel_config.pipeline_parallel_size)
131 - ]
126 + self.pp_locks: Optional[List[asyncio.Lock]] = None
127
128 super().__init__(model_config, cache_config, parallel_config,
129 scheduler_config, device_config, load_config,
vllm/executor/ray_gpu_executor.py
+9
@@ -349,6 +349,15 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
349 self,
350 execute_model_req: Optional[ExecuteModelRequest] = None
351 ) -> List[SamplerOutput]:
352 + if self.pp_locks is None:
353 + # This locks each pipeline parallel stage so multiple virtual
354 + # engines can't execute on the same stage at the same time
355 + # We create the locks here to avoid creating them in the constructor
356 + # which uses a different asyncio loop.
357 + self.pp_locks = [
358 + asyncio.Lock()
359 + for _ in range(self.parallel_config.pipeline_parallel_size)
360 + ]
361
362 async def _run_task_with_lock(task, lock, *args, **kwargs):
363 async with lock: