@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5792c7c4a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Hardware][NVIDIA] Add non-NVML CUDA mode for Jetson (#9735)

Signed-off-by: Conroy Cheers <conroy@corncheese.org>

Conroy Cheers committed Nov 27, 2024 at 05:26 UTC f5792c7c4a63ecdd2dcaa068ac7986dc4a22436b
3 files changed +155 -87
CMakeLists.txt
+5 -5
@@ -34,7 +34,7 @@ install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
34 set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12")
35
36 # Supported NVIDIA architectures.
37 -set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
37 +set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0")
38
39 # Supported AMD GPU architectures.
40 set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101")
@@ -249,7 +249,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
249 # Only build Marlin kernels if we are building for at least some compatible archs.
250 # Keep building Marlin for 9.0 as there are some group sizes and shapes that
251 # are not supported by Machete yet.
252 - cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
252 + cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.7;8.9;9.0" ${CUDA_ARCHS})
253 if (MARLIN_ARCHS)
254 set(MARLIN_SRCS
255 "csrc/quantization/fp8/fp8_marlin.cu"
@@ -300,8 +300,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
300 #
301 # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
302 # kernels for the remaining archs that are not already built for 3x.
303 - cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304 - "7.5;8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
303 + cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304 + "7.5;8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
305 # subtract out the archs that are already built for 3x
306 list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
307 if (SCALED_MM_2X_ARCHS)
@@ -427,7 +427,7 @@ set_gencode_flags_for_srcs(
427 CUDA_ARCHS "${CUDA_ARCHS}")
428
429 if(VLLM_GPU_LANG STREQUAL "CUDA")
430 - cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
430 + cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
431 if (MARLIN_MOE_ARCHS)
432 set(MARLIN_MOE_SRC
433 "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
vllm/platforms/__init__.py
+9 -1
@@ -28,7 +28,15 @@ try:
28 finally:
29 pynvml.nvmlShutdown()
30 except Exception:
31 - pass
31 + # CUDA is supported on Jetson, but NVML may not be.
32 + import os
33 +
34 + def cuda_is_jetson() -> bool:
35 + return os.path.isfile("/etc/nv_tegra_release") \
36 + or os.path.exists("/sys/class/tegra-firmware")
37 +
38 + if cuda_is_jetson():
39 + is_cuda = True
40
41 is_rocm = False
42
vllm/platforms/cuda.py
+141 -81
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import TYPE_CHECKING, Callable, List, Tuple, TypeVar
7 +from typing import TYPE_CHECKING, Callable, List, TypeVar
8
9 import pynvml
10 import torch
@@ -38,10 +38,23 @@ if pynvml.__file__.endswith("__init__.py"):
38 # see https://github.com/huggingface/diffusers/issues/9704 for details
39 torch.backends.cuda.enable_cudnn_sdp(False)
40
41 -# NVML utils
42 -# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
43 -# all the related functions work on real physical device ids.
44 -# the major benefit of using NVML is that it will not initialize CUDA
41 +
42 +def device_id_to_physical_device_id(device_id: int) -> int:
43 + if "CUDA_VISIBLE_DEVICES" in os.environ:
44 + device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
45 + if device_ids == [""]:
46 + msg = (
47 + "CUDA_VISIBLE_DEVICES is set to empty string, which means"
48 + " GPU support is disabled. If you are using ray, please unset"
49 + " the environment variable `CUDA_VISIBLE_DEVICES` inside the"
50 + " worker/actor. "
51 + "Check https://github.com/vllm-project/vllm/issues/8402 for"
52 + " more information.")
53 + raise RuntimeError(msg)
54 + physical_device_id = device_ids[device_id]
55 + return int(physical_device_id)
56 + else:
57 + return device_id
58
59
60 def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
@@ -57,87 +70,75 @@ def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
70 return wrapper
71
72
60 -@lru_cache(maxsize=8)
61 -@with_nvml_context
62 -def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
63 - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
64 - return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
65 -
66 -
67 -@lru_cache(maxsize=8)
68 -@with_nvml_context
69 -def get_physical_device_name(device_id: int = 0) -> str:
70 - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
71 - return pynvml.nvmlDeviceGetName(handle)
72 -
73 -
74 -@lru_cache(maxsize=8)
75 -@with_nvml_context
76 -def get_physical_device_total_memory(device_id: int = 0) -> int:
77 - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
78 - return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
79 -
73 +class CudaPlatformBase(Platform):
74 + _enum = PlatformEnum.CUDA
75 + device_type: str = "cuda"
76 + dispatch_key: str = "CUDA"
77
81 -@with_nvml_context
82 -def warn_if_different_devices():
83 - device_ids: int = pynvml.nvmlDeviceGetCount()
84 - if device_ids > 1:
85 - device_names = [get_physical_device_name(i) for i in range(device_ids)]
86 - if len(set(device_names)) > 1 and os.environ.get(
87 - "CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
88 - logger.warning(
89 - "Detected different devices in the system: \n%s\nPlease"
90 - " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
91 - "avoid unexpected behavior.", "\n".join(device_names))
78 + @classmethod
79 + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
80 + raise NotImplementedError
81
82 + @classmethod
83 + def get_device_name(cls, device_id: int = 0) -> str:
84 + raise NotImplementedError
85
94 -try:
95 - from sphinx.ext.autodoc.mock import _MockModule
86 + @classmethod
87 + def get_device_total_memory(cls, device_id: int = 0) -> int:
88 + raise NotImplementedError
89
97 - if not isinstance(pynvml, _MockModule):
98 - warn_if_different_devices()
99 -except ModuleNotFoundError:
100 - warn_if_different_devices()
90 + @classmethod
91 + def is_full_nvlink(cls, device_ids: List[int]) -> bool:
92 + raise NotImplementedError
93
94 + @classmethod
95 + def log_warnings(cls):
96 + pass
97
103 -def device_id_to_physical_device_id(device_id: int) -> int:
104 - if "CUDA_VISIBLE_DEVICES" in os.environ:
105 - device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
106 - if device_ids == [""]:
107 - msg = (
108 - "CUDA_VISIBLE_DEVICES is set to empty string, which means"
109 - " GPU support is disabled. If you are using ray, please unset"
110 - " the environment variable `CUDA_VISIBLE_DEVICES` inside the"
111 - " worker/actor. "
112 - "Check https://github.com/vllm-project/vllm/issues/8402 for"
113 - " more information.")
114 - raise RuntimeError(msg)
115 - physical_device_id = device_ids[device_id]
116 - return int(physical_device_id)
117 - else:
118 - return device_id
98 + @classmethod
99 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
100 + parallel_config = vllm_config.parallel_config
101 + scheduler_config = vllm_config.scheduler_config
102 + if parallel_config.worker_cls == "auto":
103 + if scheduler_config.is_multi_step:
104 + parallel_config.worker_cls = \
105 + "vllm.worker.multi_step_worker.MultiStepWorker"
106 + elif vllm_config.speculative_config:
107 + parallel_config.worker_cls = \
108 + "vllm.spec_decode.spec_decode_worker.create_spec_worker"
109 + else:
110 + parallel_config.worker_cls = "vllm.worker.worker.Worker"
111
112
121 -class CudaPlatform(Platform):
122 - _enum = PlatformEnum.CUDA
123 - device_type: str = "cuda"
124 - dispatch_key: str = "CUDA"
113 +# NVML utils
114 +# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
115 +# all the related functions work on real physical device ids.
116 +# the major benefit of using NVML is that it will not initialize CUDA
117 +class NvmlCudaPlatform(CudaPlatformBase):
118
119 @classmethod
120 + @lru_cache(maxsize=8)
121 + @with_nvml_context
122 def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
123 physical_device_id = device_id_to_physical_device_id(device_id)
129 - major, minor = get_physical_device_capability(physical_device_id)
124 + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
125 + major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
126 return DeviceCapability(major=major, minor=minor)
127
128 @classmethod
129 + @lru_cache(maxsize=8)
130 + @with_nvml_context
131 def get_device_name(cls, device_id: int = 0) -> str:
132 physical_device_id = device_id_to_physical_device_id(device_id)
135 - return get_physical_device_name(physical_device_id)
133 + return cls._get_physical_device_name(physical_device_id)
134
135 @classmethod
136 + @lru_cache(maxsize=8)
137 + @with_nvml_context
138 def get_device_total_memory(cls, device_id: int = 0) -> int:
139 physical_device_id = device_id_to_physical_device_id(device_id)
140 - return get_physical_device_total_memory(physical_device_id)
140 + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
141 + return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
142
143 @classmethod
144 @with_nvml_context
@@ -153,27 +154,86 @@ class CudaPlatform(Platform):
154 if i < j:
155 try:
156 p2p_status = pynvml.nvmlDeviceGetP2PStatus(
156 - handle, peer_handle,
157 - pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
157 + handle,
158 + peer_handle,
159 + pynvml.NVML_P2P_CAPS_INDEX_NVLINK,
160 + )
161 if p2p_status != pynvml.NVML_P2P_STATUS_OK:
162 return False
163 except pynvml.NVMLError:
164 logger.exception(
162 - "NVLink detection failed. This is normal if your"
163 - " machine has no NVLink equipped.")
165 + "NVLink detection failed. This is normal if"
166 + " your machine has no NVLink equipped.")
167 return False
168 return True
169
170 @classmethod
168 - def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
169 - parallel_config = vllm_config.parallel_config
170 - scheduler_config = vllm_config.scheduler_config
171 - if parallel_config.worker_cls == "auto":
172 - if scheduler_config.is_multi_step:
173 - parallel_config.worker_cls = \
174 - "vllm.worker.multi_step_worker.MultiStepWorker"
175 - elif vllm_config.speculative_config:
176 - parallel_config.worker_cls = \
177 - "vllm.spec_decode.spec_decode_worker.create_spec_worker"
178 - else:
179 - parallel_config.worker_cls = "vllm.worker.worker.Worker"
171 + def _get_physical_device_name(cls, device_id: int = 0) -> str:
172 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
173 + return pynvml.nvmlDeviceGetName(handle)
174 +
175 + @classmethod
176 + @with_nvml_context
177 + def log_warnings(cls):
178 + device_ids: int = pynvml.nvmlDeviceGetCount()
179 + if device_ids > 1:
180 + device_names = [
181 + cls._get_physical_device_name(i) for i in range(device_ids)
182 + ]
183 + if (len(set(device_names)) > 1
184 + and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"):
185 + logger.warning(
186 + "Detected different devices in the system: \n%s\nPlease"
187 + " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
188 + "avoid unexpected behavior.",
189 + "\n".join(device_names),
190 + )
191 +
192 +
193 +class NonNvmlCudaPlatform(CudaPlatformBase):
194 +
195 + @classmethod
196 + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
197 + major, minor = torch.cuda.get_device_capability(device_id)
198 + return DeviceCapability(major=major, minor=minor)
199 +
200 + @classmethod
201 + def get_device_name(cls, device_id: int = 0) -> str:
202 + return torch.cuda.get_device_name(device_id)
203 +
204 + @classmethod
205 + def get_device_total_memory(cls, device_id: int = 0) -> int:
206 + device_props = torch.cuda.get_device_properties(device_id)
207 + return device_props.total_memory
208 +
209 + @classmethod
210 + def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool:
211 + logger.exception(
212 + "NVLink detection not possible, as context support was"
213 + " not found. Assuming no NVLink available.")
214 + return False
215 +
216 +
217 +# Autodetect either NVML-enabled or non-NVML platform
218 +# based on whether NVML is available.
219 +nvml_available = False
220 +try:
221 + try:
222 + pynvml.nvmlInit()
223 + nvml_available = True
224 + except Exception:
225 + # On Jetson, NVML is not supported.
226 + nvml_available = False
227 +finally:
228 + if nvml_available:
229 + pynvml.nvmlShutdown()
230 +
231 +CudaPlatform = NvmlCudaPlatform if nvml_available else NonNvmlCudaPlatform
232 +
233 +try:
234 + from sphinx.ext.autodoc.mock import _MockModule
235 +
236 + if not isinstance(pynvml, _MockModule):
237 + CudaPlatform.log_warnings()
238 +except ModuleNotFoundError:
239 + CudaPlatform.log_warnings()