@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 72ff3a9686
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[core] Bump ray to use _overlap_gpu_communication in compiled graph tests (#10410)

Signed-off-by: Rui Qiao <ubuntu@ip-172-31-15-128.us-west-2.compute.internal> Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Rui Qiao <ubuntu@ip-172-31-15-128.us-west-2.compute.internal>

Rui Qiao committed Dec 11, 2024 at 11:36 UTC 72ff3a968682e6a3f7620ab59f2baf5e8eb2777b
4 files changed +20 -9
requirements-test.in
+1 -1
@@ -13,7 +13,7 @@ einops # required for MPT, qwen-vl and Mamba
13 httpx
14 librosa # required for audio tests
15 peft
16 -ray[adag]==2.35
16 +ray[adag]==2.40.0
17 sentence-transformers # required for embedding tests
18 soundfile # required for audio tests
19 timm # required for internvl test
requirements-test.txt
+1 -1
@@ -410,7 +410,7 @@ pyyaml==6.0.2
410 # ray
411 # timm
412 # transformers
413 -ray[adag]==2.35.0
413 +ray[adag]==2.40.0
414 # via -r requirements-test.in
415 redis==5.2.0
416 # via tensorizer
vllm/envs.py
+8
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
45 VLLM_USE_RAY_SPMD_WORKER: bool = False
46 VLLM_USE_RAY_COMPILED_DAG: bool = False
47 VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
48 + VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = True
49 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -337,6 +338,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
338 lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
339 ),
340
341 + # If the env var is set, it enables GPU communication overlap in
342 + # Ray's compiled DAG. This flag is ignored if
343 + # VLLM_USE_RAY_COMPILED_DAG is not set.
344 + "VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM":
345 + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "1"))
346 + ),
347 +
348 # Use dedicated multiprocess context for workers.
349 # Both spawn and fork work
350 "VLLM_WORKER_MULTIPROC_METHOD":
vllm/executor/ray_gpu_executor.py
+10 -7
@@ -414,12 +414,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
414 import pkg_resources
415 from packaging import version
416
417 - required_version = version.parse("2.35")
417 + required_version = version.parse("2.40")
418 current_version = version.parse(
419 pkg_resources.get_distribution("ray").version)
420 - # TODO: update the constraint once we adapt to the backward
421 - # incompatible API change from ray 2.36
422 - if current_version != required_version:
420 + if current_version < required_version:
421 raise ValueError(f"Ray version {required_version} is "
422 f"required, but found {current_version}")
423
@@ -445,6 +443,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
443
444 logger.info("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL = %s",
445 envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL)
446 + logger.info("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM = %s",
447 + envs.VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM)
448 with InputNode() as input_data:
449 # Example DAG: PP=2, TP=4
450 # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
@@ -480,7 +480,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
480
481 forward_dag = MultiOutputNode(outputs)
482
483 - return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
483 + return forward_dag.experimental_compile(
484 + enable_asyncio=enable_asyncio,
485 + _overlap_gpu_communication=envs.
486 + VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM)
487
488 def __del__(self):
489 self.shutdown()
@@ -507,8 +510,8 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
510
511 serialized_data = self.input_encoder.encode(execute_model_req)
512 dag_future = await self.forward_dag.execute_async(serialized_data)
510 - outputs = await dag_future
511 - return self.output_decoder.decode(outputs[0])
513 + output = await dag_future[0]
514 + return self.output_decoder.decode(output)
515
516 async def _driver_execute_model_async(
517 self,