@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18e60d7d13
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[misc][distributed] add VLLM_SKIP_P2P_CHECK flag (#8911)

youkaichao committed Sep 27, 2024 at 14:27 UTC 18e60d7d1394541b48bf48b0a57a546a93607ac2
2 files changed +12
vllm/distributed/device_communicators/custom_all_reduce.py
+4
@@ -28,6 +28,10 @@ def _can_p2p(rank: int, world_size: int) -> bool:
28 for i in range(world_size):
29 if i == rank:
30 continue
31 + if envs.VLLM_SKIP_P2P_CHECK:
32 + logger.info(
33 + "Skipping P2P check and trusting the driver's P2P report.")
34 + return torch.cuda.can_device_access_peer(rank, i)
35 if not gpu_p2p_access_check(rank, i):
36 return False
37 return True
vllm/envs.py
+8
@@ -63,6 +63,7 @@ if TYPE_CHECKING:
63 VLLM_USE_TRITON_AWQ: bool = False
64 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
65 VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False
66 + VLLM_SKIP_P2P_CHECK: bool = False
67
68
69 def get_default_cache_root():
@@ -423,6 +424,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
424 lambda:
425 (os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in
426 ("1", "true")),
427 +
428 + # By default, vLLM will check the peer-to-peer capability itself,
429 + # in case of broken drivers. See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa
430 + # If this env var is set to 1, vLLM will skip the peer-to-peer check,
431 + # and trust the driver's peer-to-peer capability report.
432 + "VLLM_SKIP_P2P_CHECK":
433 + lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
434 }
435
436 # end-env-vars-definition