A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Distributed][Core] Support Py39 and Py38 for PP (#6120)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Murali Andoorveedu committed
Jul 3, 2024 at 17:52 UTC
0ed646b7aa3b434b2fcb6f6b6e725570879cb89e
2 files changed
+10
-6
vllm/executor/executor_base.py
+1
-6
@@ -123,12 +123,7 @@ class ExecutorAsyncBase(ExecutorBase):
123
multimodal_config: Optional[MultiModalConfig],
124
speculative_config: Optional[SpeculativeConfig],
125
) -> None:
126
- # This locks each pipeline parallel stage so multiple virtual engines
127
- # can't execute on the same stage at the same time
128
- self.pp_locks = [
129
- asyncio.Lock()
130
- for _ in range(parallel_config.pipeline_parallel_size)
131
- ]
126
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
127
128
super().__init__(model_config, cache_config, parallel_config,
129
scheduler_config, device_config, load_config,
vllm/executor/ray_gpu_executor.py
+9
@@ -349,6 +349,15 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
349
self,
350
execute_model_req: Optional[ExecuteModelRequest] = None
351
) -> List[SamplerOutput]:
352
+ if self.pp_locks is None:
353
+ # This locks each pipeline parallel stage so multiple virtual
354
+ # engines can't execute on the same stage at the same time
355
+ # We create the locks here to avoid creating them in the constructor
356
+ # which uses a different asyncio loop.
357
+ self.pp_locks = [
358
+ asyncio.Lock()
359
+ for _ in range(self.parallel_config.pipeline_parallel_size)
360
+ ]
361
362
async def _run_task_with_lock(task, lock, *args, **kwargs):
363
async with lock: