A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[core] Bump ray to use _overlap_gpu_communication in compiled graph tests (#10410)
Signed-off-by: Rui Qiao <ubuntu@ip-172-31-15-128.us-west-2.compute.internal> Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Rui Qiao <ubuntu@ip-172-31-15-128.us-west-2.compute.internal>
Rui Qiao committed
Dec 11, 2024 at 11:36 UTC
72ff3a968682e6a3f7620ab59f2baf5e8eb2777b
4 files changed
+20
-9
requirements-test.in
+1
-1
@@ -13,7 +13,7 @@ einops # required for MPT, qwen-vl and Mamba
13
httpx
14
librosa # required for audio tests
15
peft
16
-ray[adag]==2.35
16
+ray[adag]==2.40.0
17
sentence-transformers # required for embedding tests
18
soundfile # required for audio tests
19
timm # required for internvl test
requirements-test.txt
+1
-1
@@ -410,7 +410,7 @@ pyyaml==6.0.2
410
# ray
411
# timm
412
# transformers
413
-ray[adag]==2.35.0
413
+ray[adag]==2.40.0
414
# via -r requirements-test.in
415
redis==5.2.0
416
# via tensorizer
vllm/envs.py
+8
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
45
VLLM_USE_RAY_SPMD_WORKER: bool = False
46
VLLM_USE_RAY_COMPILED_DAG: bool = False
47
VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
48
+ VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = True
49
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -337,6 +338,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
338
lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
339
),
340
341
+ # If the env var is set, it enables GPU communication overlap in
342
+ # Ray's compiled DAG. This flag is ignored if
343
+ # VLLM_USE_RAY_COMPILED_DAG is not set.
344
+ "VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM":
345
+ lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "1"))
346
+ ),
347
+
348
# Use dedicated multiprocess context for workers.
349
# Both spawn and fork work
350
"VLLM_WORKER_MULTIPROC_METHOD":
vllm/executor/ray_gpu_executor.py
+10
-7
@@ -414,12 +414,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
414
import pkg_resources
415
from packaging import version
416
417
- required_version = version.parse("2.35")
417
+ required_version = version.parse("2.40")
418
current_version = version.parse(
419
pkg_resources.get_distribution("ray").version)
420
- # TODO: update the constraint once we adapt to the backward
421
- # incompatible API change from ray 2.36
422
- if current_version != required_version:
420
+ if current_version < required_version:
421
raise ValueError(f"Ray version {required_version} is "
422
f"required, but found {current_version}")
423
@@ -445,6 +443,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
443
444
logger.info("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL = %s",
445
envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL)
446
+ logger.info("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM = %s",
447
+ envs.VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM)
448
with InputNode() as input_data:
449
# Example DAG: PP=2, TP=4
450
# (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
@@ -480,7 +480,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
480
481
forward_dag = MultiOutputNode(outputs)
482
483
- return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
483
+ return forward_dag.experimental_compile(
484
+ enable_asyncio=enable_asyncio,
485
+ _overlap_gpu_communication=envs.
486
+ VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM)
487
488
def __del__(self):
489
self.shutdown()
@@ -507,8 +510,8 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
510
511
serialized_data = self.input_encoder.encode(execute_model_req)
512
dag_future = await self.forward_dag.execute_async(serialized_data)
510
- outputs = await dag_future
511
- return self.output_decoder.decode(outputs[0])
513
+ output = await dag_future[0]
514
+ return self.output_decoder.decode(output)
515
516
async def _driver_execute_model_async(
517
self,