@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18473cf498
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Neuron] Add an option to build with neuron (#2065)

Liangfu Chen committed Jan 18, 2024 at 10:58 UTC 18473cf498615a68b88a5b2c5202e221e1db87ad
3 files changed +64 -13
requirements-neuron.txt new
+9
@@ -0,0 +1,9 @@
1 +sentencepiece # Required for LLaMA tokenizer.
2 +numpy
3 +transformers-neuronx >= 0.9.0
4 +torch-neuronx >= 2.1.0
5 +neuronx-cc
6 +fastapi
7 +uvicorn[standard]
8 +pydantic == 1.10.13 # Required for OpenAI server.
9 +aioprometheus[starlette]
setup.py
+51 -11
@@ -24,8 +24,17 @@ def _is_hip() -> bool:
24 return torch.version.hip is not None
25
26
27 +def _is_neuron() -> bool:
28 + torch_neuronx_installed = True
29 + try:
30 + subprocess.run(["neuron-ls"], capture_output=True, check=True)
31 + except FileNotFoundError as e:
32 + torch_neuronx_installed = False
33 + return torch_neuronx_installed
34 +
35 +
36 def _is_cuda() -> bool:
28 - return torch.version.cuda is not None
37 + return (torch.version.cuda is not None) and not _is_neuron()
38
39
40 # Compiler flags.
@@ -87,6 +96,24 @@ def get_hipcc_rocm_version():
96 return None
97
98
99 +def get_neuronxcc_version():
100 + import sysconfig
101 + site_dir = sysconfig.get_paths()["purelib"]
102 + version_file = os.path.join(site_dir, "neuronxcc", "version", "__init__.py")
103 +
104 + # Check if the command was executed successfully
105 + with open(version_file, "rt") as fp:
106 + content = fp.read()
107 +
108 + # Extract the version using a regular expression
109 + match = re.search(r"__version__ = '(\S+)'", content)
110 + if match:
111 + # Return the version string
112 + return match.group(1)
113 + else:
114 + raise RuntimeError("Could not find HIP version in the output")
115 +
116 +
117 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
118 """Get the CUDA version from nvcc.
119
@@ -210,6 +237,9 @@ elif _is_hip():
237 f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
238 f"amdgpu_arch_found: {amd_arch}")
239
240 +elif _is_neuron():
241 + neuronxcc_version = get_neuronxcc_version()
242 +
243 ext_modules = []
244
245 vllm_extension_sources = [
@@ -227,15 +257,16 @@ vllm_extension_sources = [
257 if _is_cuda():
258 vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
259
230 -vllm_extension = CUDAExtension(
231 - name="vllm._C",
232 - sources=vllm_extension_sources,
233 - extra_compile_args={
234 - "cxx": CXX_FLAGS,
235 - "nvcc": NVCC_FLAGS,
236 - },
237 -)
238 -ext_modules.append(vllm_extension)
260 +if not _is_neuron():
261 + vllm_extension = CUDAExtension(
262 + name="vllm._C",
263 + sources=vllm_extension_sources,
264 + extra_compile_args={
265 + "cxx": CXX_FLAGS,
266 + "nvcc": NVCC_FLAGS,
267 + },
268 + )
269 + ext_modules.append(vllm_extension)
270
271
272 def get_path(*filepath) -> str:
@@ -264,6 +295,12 @@ def get_vllm_version() -> str:
295 if hipcc_version != MAIN_CUDA_VERSION:
296 rocm_version_str = hipcc_version.replace(".", "")[:3]
297 version += f"+rocm{rocm_version_str}"
298 + elif _is_neuron():
299 + # Get the Neuron version
300 + neuron_version = str(neuronxcc_version)
301 + if neuron_version != MAIN_CUDA_VERSION:
302 + neuron_version_str = neuron_version.replace(".", "")[:3]
303 + version += f"+neuron{neuron_version_str}"
304 else:
305 cuda_version = str(nvcc_cuda_version)
306 if cuda_version != MAIN_CUDA_VERSION:
@@ -287,6 +324,9 @@ def get_requirements() -> List[str]:
324 if _is_hip():
325 with open(get_path("requirements-rocm.txt")) as f:
326 requirements = f.read().strip().split("\n")
327 + elif _is_neuron():
328 + with open(get_path("requirements-neuron.txt")) as f:
329 + requirements = f.read().strip().split("\n")
330 else:
331 with open(get_path("requirements.txt")) as f:
332 requirements = f.read().strip().split("\n")
@@ -325,6 +365,6 @@ setuptools.setup(
365 python_requires=">=3.8",
366 install_requires=get_requirements(),
367 ext_modules=ext_modules,
328 - cmdclass={"build_ext": BuildExtension},
368 + cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
369 package_data=package_data,
370 )
vllm/utils.py
+4 -2
@@ -8,8 +8,6 @@ from typing import List
8 import psutil
9 import torch
10
11 -from vllm._C import cuda_utils
12 -
11
12 class Device(enum.Enum):
13 GPU = enum.auto()
@@ -36,6 +34,10 @@ def is_hip() -> bool:
34
35 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
36 """Returns the maximum shared memory per thread block in bytes."""
37 + # NOTE: This import statement should be executed lazily since
38 + # the Neuron-X backend does not have the `cuda_utils` module.
39 + from vllm._C import cuda_utils
40 +
41 # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
42 cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
43 max_shared_mem = cuda_utils.get_device_attribute(