A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Hardware][NVIDIA] Add non-NVML CUDA mode for Jetson (#9735)
Signed-off-by: Conroy Cheers <conroy@corncheese.org>
Conroy Cheers committed
Nov 27, 2024 at 05:26 UTC
f5792c7c4a63ecdd2dcaa068ac7986dc4a22436b
3 files changed
+155
-87
CMakeLists.txt
+5
-5
@@ -34,7 +34,7 @@ install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
34
set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12")
35
36
# Supported NVIDIA architectures.
37
-set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
37
+set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0")
38
39
# Supported AMD GPU architectures.
40
set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101")
@@ -249,7 +249,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
249
# Only build Marlin kernels if we are building for at least some compatible archs.
250
# Keep building Marlin for 9.0 as there are some group sizes and shapes that
251
# are not supported by Machete yet.
252
- cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
252
+ cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.7;8.9;9.0" ${CUDA_ARCHS})
253
if (MARLIN_ARCHS)
254
set(MARLIN_SRCS
255
"csrc/quantization/fp8/fp8_marlin.cu"
@@ -300,8 +300,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
300
#
301
# For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
302
# kernels for the remaining archs that are not already built for 3x.
303
- cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304
- "7.5;8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
303
+ cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304
+ "7.5;8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
305
# subtract out the archs that are already built for 3x
306
list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
307
if (SCALED_MM_2X_ARCHS)
@@ -427,7 +427,7 @@ set_gencode_flags_for_srcs(
427
CUDA_ARCHS "${CUDA_ARCHS}")
428
429
if(VLLM_GPU_LANG STREQUAL "CUDA")
430
- cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
430
+ cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
431
if (MARLIN_MOE_ARCHS)
432
set(MARLIN_MOE_SRC
433
"csrc/moe/marlin_kernels/marlin_moe_kernel.h"
vllm/platforms/__init__.py
+9
-1
@@ -28,7 +28,15 @@ try:
28
finally:
29
pynvml.nvmlShutdown()
30
except Exception:
31
- pass
31
+ # CUDA is supported on Jetson, but NVML may not be.
32
+ import os
33
+
34
+ def cuda_is_jetson() -> bool:
35
+ return os.path.isfile("/etc/nv_tegra_release") \
36
+ or os.path.exists("/sys/class/tegra-firmware")
37
+
38
+ if cuda_is_jetson():
39
+ is_cuda = True
40
41
is_rocm = False
42
vllm/platforms/cuda.py
+141
-81
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import TYPE_CHECKING, Callable, List, Tuple, TypeVar
7
+from typing import TYPE_CHECKING, Callable, List, TypeVar
8
9
import pynvml
10
import torch
@@ -38,10 +38,23 @@ if pynvml.__file__.endswith("__init__.py"):
38
# see https://github.com/huggingface/diffusers/issues/9704 for details
39
torch.backends.cuda.enable_cudnn_sdp(False)
40
41
-# NVML utils
42
-# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
43
-# all the related functions work on real physical device ids.
44
-# the major benefit of using NVML is that it will not initialize CUDA
41
+
42
+def device_id_to_physical_device_id(device_id: int) -> int:
43
+ if "CUDA_VISIBLE_DEVICES" in os.environ:
44
+ device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
45
+ if device_ids == [""]:
46
+ msg = (
47
+ "CUDA_VISIBLE_DEVICES is set to empty string, which means"
48
+ " GPU support is disabled. If you are using ray, please unset"
49
+ " the environment variable `CUDA_VISIBLE_DEVICES` inside the"
50
+ " worker/actor. "
51
+ "Check https://github.com/vllm-project/vllm/issues/8402 for"
52
+ " more information.")
53
+ raise RuntimeError(msg)
54
+ physical_device_id = device_ids[device_id]
55
+ return int(physical_device_id)
56
+ else:
57
+ return device_id
58
59
60
def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
@@ -57,87 +70,75 @@ def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
70
return wrapper
71
72
60
-@lru_cache(maxsize=8)
61
-@with_nvml_context
62
-def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
63
- handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
64
- return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
65
-
66
-
67
-@lru_cache(maxsize=8)
68
-@with_nvml_context
69
-def get_physical_device_name(device_id: int = 0) -> str:
70
- handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
71
- return pynvml.nvmlDeviceGetName(handle)
72
-
73
-
74
-@lru_cache(maxsize=8)
75
-@with_nvml_context
76
-def get_physical_device_total_memory(device_id: int = 0) -> int:
77
- handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
78
- return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
79
-
73
+class CudaPlatformBase(Platform):
74
+ _enum = PlatformEnum.CUDA
75
+ device_type: str = "cuda"
76
+ dispatch_key: str = "CUDA"
77
81
-@with_nvml_context
82
-def warn_if_different_devices():
83
- device_ids: int = pynvml.nvmlDeviceGetCount()
84
- if device_ids > 1:
85
- device_names = [get_physical_device_name(i) for i in range(device_ids)]
86
- if len(set(device_names)) > 1 and os.environ.get(
87
- "CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
88
- logger.warning(
89
- "Detected different devices in the system: \n%s\nPlease"
90
- " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
91
- "avoid unexpected behavior.", "\n".join(device_names))
78
+ @classmethod
79
+ def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
80
+ raise NotImplementedError
81
82
+ @classmethod
83
+ def get_device_name(cls, device_id: int = 0) -> str:
84
+ raise NotImplementedError
85
94
-try:
95
- from sphinx.ext.autodoc.mock import _MockModule
86
+ @classmethod
87
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
88
+ raise NotImplementedError
89
97
- if not isinstance(pynvml, _MockModule):
98
- warn_if_different_devices()
99
-except ModuleNotFoundError:
100
- warn_if_different_devices()
90
+ @classmethod
91
+ def is_full_nvlink(cls, device_ids: List[int]) -> bool:
92
+ raise NotImplementedError
93
94
+ @classmethod
95
+ def log_warnings(cls):
96
+ pass
97
103
-def device_id_to_physical_device_id(device_id: int) -> int:
104
- if "CUDA_VISIBLE_DEVICES" in os.environ:
105
- device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
106
- if device_ids == [""]:
107
- msg = (
108
- "CUDA_VISIBLE_DEVICES is set to empty string, which means"
109
- " GPU support is disabled. If you are using ray, please unset"
110
- " the environment variable `CUDA_VISIBLE_DEVICES` inside the"
111
- " worker/actor. "
112
- "Check https://github.com/vllm-project/vllm/issues/8402 for"
113
- " more information.")
114
- raise RuntimeError(msg)
115
- physical_device_id = device_ids[device_id]
116
- return int(physical_device_id)
117
- else:
118
- return device_id
98
+ @classmethod
99
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
100
+ parallel_config = vllm_config.parallel_config
101
+ scheduler_config = vllm_config.scheduler_config
102
+ if parallel_config.worker_cls == "auto":
103
+ if scheduler_config.is_multi_step:
104
+ parallel_config.worker_cls = \
105
+ "vllm.worker.multi_step_worker.MultiStepWorker"
106
+ elif vllm_config.speculative_config:
107
+ parallel_config.worker_cls = \
108
+ "vllm.spec_decode.spec_decode_worker.create_spec_worker"
109
+ else:
110
+ parallel_config.worker_cls = "vllm.worker.worker.Worker"
111
112
121
-class CudaPlatform(Platform):
122
- _enum = PlatformEnum.CUDA
123
- device_type: str = "cuda"
124
- dispatch_key: str = "CUDA"
113
+# NVML utils
114
+# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
115
+# all the related functions work on real physical device ids.
116
+# the major benefit of using NVML is that it will not initialize CUDA
117
+class NvmlCudaPlatform(CudaPlatformBase):
118
119
@classmethod
120
+ @lru_cache(maxsize=8)
121
+ @with_nvml_context
122
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
123
physical_device_id = device_id_to_physical_device_id(device_id)
129
- major, minor = get_physical_device_capability(physical_device_id)
124
+ handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
125
+ major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
126
return DeviceCapability(major=major, minor=minor)
127
128
@classmethod
129
+ @lru_cache(maxsize=8)
130
+ @with_nvml_context
131
def get_device_name(cls, device_id: int = 0) -> str:
132
physical_device_id = device_id_to_physical_device_id(device_id)
135
- return get_physical_device_name(physical_device_id)
133
+ return cls._get_physical_device_name(physical_device_id)
134
135
@classmethod
136
+ @lru_cache(maxsize=8)
137
+ @with_nvml_context
138
def get_device_total_memory(cls, device_id: int = 0) -> int:
139
physical_device_id = device_id_to_physical_device_id(device_id)
140
- return get_physical_device_total_memory(physical_device_id)
140
+ handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
141
+ return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
142
143
@classmethod
144
@with_nvml_context
@@ -153,27 +154,86 @@ class CudaPlatform(Platform):
154
if i < j:
155
try:
156
p2p_status = pynvml.nvmlDeviceGetP2PStatus(
156
- handle, peer_handle,
157
- pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
157
+ handle,
158
+ peer_handle,
159
+ pynvml.NVML_P2P_CAPS_INDEX_NVLINK,
160
+ )
161
if p2p_status != pynvml.NVML_P2P_STATUS_OK:
162
return False
163
except pynvml.NVMLError:
164
logger.exception(
162
- "NVLink detection failed. This is normal if your"
163
- " machine has no NVLink equipped.")
165
+ "NVLink detection failed. This is normal if"
166
+ " your machine has no NVLink equipped.")
167
return False
168
return True
169
170
@classmethod
168
- def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
169
- parallel_config = vllm_config.parallel_config
170
- scheduler_config = vllm_config.scheduler_config
171
- if parallel_config.worker_cls == "auto":
172
- if scheduler_config.is_multi_step:
173
- parallel_config.worker_cls = \
174
- "vllm.worker.multi_step_worker.MultiStepWorker"
175
- elif vllm_config.speculative_config:
176
- parallel_config.worker_cls = \
177
- "vllm.spec_decode.spec_decode_worker.create_spec_worker"
178
- else:
179
- parallel_config.worker_cls = "vllm.worker.worker.Worker"
171
+ def _get_physical_device_name(cls, device_id: int = 0) -> str:
172
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
173
+ return pynvml.nvmlDeviceGetName(handle)
174
+
175
+ @classmethod
176
+ @with_nvml_context
177
+ def log_warnings(cls):
178
+ device_ids: int = pynvml.nvmlDeviceGetCount()
179
+ if device_ids > 1:
180
+ device_names = [
181
+ cls._get_physical_device_name(i) for i in range(device_ids)
182
+ ]
183
+ if (len(set(device_names)) > 1
184
+ and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"):
185
+ logger.warning(
186
+ "Detected different devices in the system: \n%s\nPlease"
187
+ " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
188
+ "avoid unexpected behavior.",
189
+ "\n".join(device_names),
190
+ )
191
+
192
+
193
+class NonNvmlCudaPlatform(CudaPlatformBase):
194
+
195
+ @classmethod
196
+ def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
197
+ major, minor = torch.cuda.get_device_capability(device_id)
198
+ return DeviceCapability(major=major, minor=minor)
199
+
200
+ @classmethod
201
+ def get_device_name(cls, device_id: int = 0) -> str:
202
+ return torch.cuda.get_device_name(device_id)
203
+
204
+ @classmethod
205
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
206
+ device_props = torch.cuda.get_device_properties(device_id)
207
+ return device_props.total_memory
208
+
209
+ @classmethod
210
+ def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool:
211
+ logger.exception(
212
+ "NVLink detection not possible, as context support was"
213
+ " not found. Assuming no NVLink available.")
214
+ return False
215
+
216
+
217
+# Autodetect either NVML-enabled or non-NVML platform
218
+# based on whether NVML is available.
219
+nvml_available = False
220
+try:
221
+ try:
222
+ pynvml.nvmlInit()
223
+ nvml_available = True
224
+ except Exception:
225
+ # On Jetson, NVML is not supported.
226
+ nvml_available = False
227
+finally:
228
+ if nvml_available:
229
+ pynvml.nvmlShutdown()
230
+
231
+CudaPlatform = NvmlCudaPlatform if nvml_available else NonNvmlCudaPlatform
232
+
233
+try:
234
+ from sphinx.ext.autodoc.mock import _MockModule
235
+
236
+ if not isinstance(pynvml, _MockModule):
237
+ CudaPlatform.log_warnings()
238
+except ModuleNotFoundError:
239
+ CudaPlatform.log_warnings()