A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix for multinode crash on 4 PP (#6495)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Murali Andoorveedu committed
Jul 17, 2024 at 04:25 UTC
5fa6e9876ed8212d74a71a7df7654fd4b94dd659
2 files changed
+17
-5
tests/distributed/test_pipeline_parallel.py
+3
-5
@@ -4,14 +4,12 @@ from ..utils import RemoteOpenAIServer
4
5
6
@pytest.mark.parametrize(
7
- "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME",
8
- [
7
+ "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8
(2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9
(2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10
(1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
12
- # TODO: figure out why PP=4 tests are flaky
13
- # (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
14
- # (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
11
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
13
])
14
def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
15
pp_args = [
vllm/executor/ray_gpu_executor.py
+14
@@ -224,13 +224,27 @@ class RayGPUExecutor(DistributedGPUExecutor):
224
# broadcasted to.
225
self.non_driver_workers: List[RayWorkerWrapper] = []
226
227
+ tp_driver_worker_ranks = []
228
+ non_driver_worker_ranks = []
229
for idx, rank in enumerate(worker_ranks[1:]):
230
# We need to skip the driver worker, which we
231
# do by skipping worker_ranks[0] which is always 0.
232
if rank % self.parallel_config.tensor_parallel_size == 0:
233
self.tp_driver_workers.append(self.workers[idx])
234
+ tp_driver_worker_ranks.append(rank)
235
else:
236
self.non_driver_workers.append(self.workers[idx])
237
+ non_driver_worker_ranks.append(rank)
238
+
239
+ # Enforce rank order for correct rank to return final output.
240
+ self.tp_driver_workers = [
241
+ worker for _, worker in sorted(
242
+ zip(tp_driver_worker_ranks, self.tp_driver_workers))
243
+ ]
244
+ self.non_driver_workers = [
245
+ worker for _, worker in sorted(
246
+ zip(non_driver_worker_ranks, self.non_driver_workers))
247
+ ]
248
249
def _driver_execute_model(
250
self, execute_model_req: Optional[ExecuteModelRequest]