@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 458e63a2c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[platform] add device_control env var (#12009)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 13, 2025 at 20:59 UTC 458e63a2c6b18e7febfa30cecb59461f96fbe324
7 files changed +17
vllm/platforms/cuda.py
+1
@@ -78,6 +78,7 @@ class CudaPlatformBase(Platform):
78 device_type: str = "cuda"
79 dispatch_key: str = "CUDA"
80 ray_device_key: str = "GPU"
81 + device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
82
83 @classmethod
84 def get_device_capability(cls,
vllm/platforms/hpu.py
+1
@@ -20,6 +20,7 @@ class HpuPlatform(Platform):
20 device_type: str = "hpu"
21 dispatch_key: str = "HPU"
22 ray_device_key: str = "HPU"
23 + device_control_env_var: str = "HABANA_VISIBLE_MODULES"
24
25 @classmethod
26 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/platforms/interface.py
+10
@@ -78,20 +78,30 @@ class Platform:
78 _enum: PlatformEnum
79 device_name: str
80 device_type: str
81 +
82 # available dispatch keys:
83 # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
84 # use "CPU" as a fallback for platforms not registered in PyTorch
85 dispatch_key: str = "CPU"
86 +
87 # available ray device keys:
88 # https://github.com/ray-project/ray/blob/10ba5adadcc49c60af2c358a33bb943fb491a171/python/ray/_private/ray_constants.py#L438 # noqa
89 # empty string means the device does not support ray
90 ray_device_key: str = ""
91 +
92 + # platform-agnostic way to specify the device control environment variable,
93 + # .e.g. CUDA_VISIBLE_DEVICES for CUDA.
94 + # hint: search for "get_visible_accelerator_ids_env_var" in
95 + # https://github.com/ray-project/ray/tree/master/python/ray/_private/accelerators # noqa
96 + device_control_env_var: str = "VLLM_DEVICE_CONTROL_ENV_VAR_PLACEHOLDER"
97 +
98 # The torch.compile backend for compiling simple and
99 # standalone functions. The default value is "inductor" to keep
100 # the same behavior as PyTorch.
101 # NOTE: for the forward part of the model, vLLM has another separate
102 # compilation strategy.
103 simple_compile_backend: str = "inductor"
104 +
105 supported_quantization: list[str] = []
106
107 def is_cuda(self) -> bool:
vllm/platforms/neuron.py
+1
@@ -18,6 +18,7 @@ class NeuronPlatform(Platform):
18 device_type: str = "neuron"
19 ray_device_key: str = "neuron_cores"
20 supported_quantization: list[str] = ["neuron_quant"]
21 + device_control_env_var: str = "NEURON_RT_VISIBLE_CORES"
22
23 @classmethod
24 def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/rocm.py
+2
@@ -65,6 +65,8 @@ class RocmPlatform(Platform):
65 device_type: str = "cuda"
66 dispatch_key: str = "CUDA"
67 ray_device_key: str = "GPU"
68 + # rocm shares the same device control env var as CUDA
69 + device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
70
71 supported_quantization: list[str] = [
72 "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
vllm/platforms/tpu.py
+1
@@ -20,6 +20,7 @@ class TpuPlatform(Platform):
20 device_type: str = "tpu"
21 dispatch_key: str = "XLA"
22 ray_device_key: str = "TPU"
23 + device_control_env_var: str = "TPU_VISIBLE_CHIPS"
24
25 supported_quantization: list[str] = [
26 "tpu_int8", "compressed-tensors", "compressed_tensors"
vllm/platforms/xpu.py
+1
@@ -22,6 +22,7 @@ class XPUPlatform(Platform):
22 # Intel XPU's device key is "GPU" for Ray.
23 # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
24 ray_device_key: str = "GPU"
25 + device_control_env_var: str = "ONEAPI_DEVICE_SELECTOR"
26
27 @classmethod
28 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,