@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5fa6e9876e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix for multinode crash on 4 PP (#6495)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

Murali Andoorveedu committed Jul 17, 2024 at 04:25 UTC 5fa6e9876ed8212d74a71a7df7654fd4b94dd659
2 files changed +17 -5
tests/distributed/test_pipeline_parallel.py
+3 -5
@@ -4,14 +4,12 @@ from ..utils import RemoteOpenAIServer
4
5
6 @pytest.mark.parametrize(
7 - "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME",
8 - [
7 + "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8 (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9 (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10 (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
12 - # TODO: figure out why PP=4 tests are flaky
13 - # (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
14 - # (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
11 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
13 ])
14 def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
15 pp_args = [
vllm/executor/ray_gpu_executor.py
+14
@@ -224,13 +224,27 @@ class RayGPUExecutor(DistributedGPUExecutor):
224 # broadcasted to.
225 self.non_driver_workers: List[RayWorkerWrapper] = []
226
227 + tp_driver_worker_ranks = []
228 + non_driver_worker_ranks = []
229 for idx, rank in enumerate(worker_ranks[1:]):
230 # We need to skip the driver worker, which we
231 # do by skipping worker_ranks[0] which is always 0.
232 if rank % self.parallel_config.tensor_parallel_size == 0:
233 self.tp_driver_workers.append(self.workers[idx])
234 + tp_driver_worker_ranks.append(rank)
235 else:
236 self.non_driver_workers.append(self.workers[idx])
237 + non_driver_worker_ranks.append(rank)
238 +
239 + # Enforce rank order for correct rank to return final output.
240 + self.tp_driver_workers = [
241 + worker for _, worker in sorted(
242 + zip(tp_driver_worker_ranks, self.tp_driver_workers))
243 + ]
244 + self.non_driver_workers = [
245 + worker for _, worker in sorted(
246 + zip(non_driver_worker_ranks, self.non_driver_workers))
247 + ]
248
249 def _driver_execute_model(
250 self, execute_model_req: Optional[ExecuteModelRequest]