A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[platform] add device_control env var (#12009)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 13, 2025 at 20:59 UTC
458e63a2c6b18e7febfa30cecb59461f96fbe324
7 files changed
+17
vllm/platforms/cuda.py
+1
@@ -78,6 +78,7 @@ class CudaPlatformBase(Platform):
78
device_type: str = "cuda"
79
dispatch_key: str = "CUDA"
80
ray_device_key: str = "GPU"
81
+ device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
82
83
@classmethod
84
def get_device_capability(cls,
vllm/platforms/hpu.py
+1
@@ -20,6 +20,7 @@ class HpuPlatform(Platform):
20
device_type: str = "hpu"
21
dispatch_key: str = "HPU"
22
ray_device_key: str = "HPU"
23
+ device_control_env_var: str = "HABANA_VISIBLE_MODULES"
24
25
@classmethod
26
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/platforms/interface.py
+10
@@ -78,20 +78,30 @@ class Platform:
78
_enum: PlatformEnum
79
device_name: str
80
device_type: str
81
+
82
# available dispatch keys:
83
# check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
84
# use "CPU" as a fallback for platforms not registered in PyTorch
85
dispatch_key: str = "CPU"
86
+
87
# available ray device keys:
88
# https://github.com/ray-project/ray/blob/10ba5adadcc49c60af2c358a33bb943fb491a171/python/ray/_private/ray_constants.py#L438 # noqa
89
# empty string means the device does not support ray
90
ray_device_key: str = ""
91
+
92
+ # platform-agnostic way to specify the device control environment variable,
93
+ # .e.g. CUDA_VISIBLE_DEVICES for CUDA.
94
+ # hint: search for "get_visible_accelerator_ids_env_var" in
95
+ # https://github.com/ray-project/ray/tree/master/python/ray/_private/accelerators # noqa
96
+ device_control_env_var: str = "VLLM_DEVICE_CONTROL_ENV_VAR_PLACEHOLDER"
97
+
98
# The torch.compile backend for compiling simple and
99
# standalone functions. The default value is "inductor" to keep
100
# the same behavior as PyTorch.
101
# NOTE: for the forward part of the model, vLLM has another separate
102
# compilation strategy.
103
simple_compile_backend: str = "inductor"
104
+
105
supported_quantization: list[str] = []
106
107
def is_cuda(self) -> bool:
vllm/platforms/neuron.py
+1
@@ -18,6 +18,7 @@ class NeuronPlatform(Platform):
18
device_type: str = "neuron"
19
ray_device_key: str = "neuron_cores"
20
supported_quantization: list[str] = ["neuron_quant"]
21
+ device_control_env_var: str = "NEURON_RT_VISIBLE_CORES"
22
23
@classmethod
24
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/rocm.py
+2
@@ -65,6 +65,8 @@ class RocmPlatform(Platform):
65
device_type: str = "cuda"
66
dispatch_key: str = "CUDA"
67
ray_device_key: str = "GPU"
68
+ # rocm shares the same device control env var as CUDA
69
+ device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
70
71
supported_quantization: list[str] = [
72
"awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
vllm/platforms/tpu.py
+1
@@ -20,6 +20,7 @@ class TpuPlatform(Platform):
20
device_type: str = "tpu"
21
dispatch_key: str = "XLA"
22
ray_device_key: str = "TPU"
23
+ device_control_env_var: str = "TPU_VISIBLE_CHIPS"
24
25
supported_quantization: list[str] = [
26
"tpu_int8", "compressed-tensors", "compressed_tensors"
vllm/platforms/xpu.py
+1
@@ -22,6 +22,7 @@ class XPUPlatform(Platform):
22
# Intel XPU's device key is "GPU" for Ray.
23
# see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
24
ray_device_key: str = "GPU"
25
+ device_control_env_var: str = "ONEAPI_DEVICE_SELECTOR"
26
27
@classmethod
28
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,