A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Neuron] Add an option to build with neuron (#2065)
Liangfu Chen committed
Jan 18, 2024 at 10:58 UTC
18473cf498615a68b88a5b2c5202e221e1db87ad
3 files changed
+64
-13
requirements-neuron.txt
new
+9
@@ -0,0 +1,9 @@
1
+sentencepiece # Required for LLaMA tokenizer.
2
+numpy
3
+transformers-neuronx >= 0.9.0
4
+torch-neuronx >= 2.1.0
5
+neuronx-cc
6
+fastapi
7
+uvicorn[standard]
8
+pydantic == 1.10.13 # Required for OpenAI server.
9
+aioprometheus[starlette]
setup.py
+51
-11
@@ -24,8 +24,17 @@ def _is_hip() -> bool:
24
return torch.version.hip is not None
25
26
27
+def _is_neuron() -> bool:
28
+ torch_neuronx_installed = True
29
+ try:
30
+ subprocess.run(["neuron-ls"], capture_output=True, check=True)
31
+ except FileNotFoundError as e:
32
+ torch_neuronx_installed = False
33
+ return torch_neuronx_installed
34
+
35
+
36
def _is_cuda() -> bool:
28
- return torch.version.cuda is not None
37
+ return (torch.version.cuda is not None) and not _is_neuron()
38
39
40
# Compiler flags.
@@ -87,6 +96,24 @@ def get_hipcc_rocm_version():
96
return None
97
98
99
+def get_neuronxcc_version():
100
+ import sysconfig
101
+ site_dir = sysconfig.get_paths()["purelib"]
102
+ version_file = os.path.join(site_dir, "neuronxcc", "version", "__init__.py")
103
+
104
+ # Check if the command was executed successfully
105
+ with open(version_file, "rt") as fp:
106
+ content = fp.read()
107
+
108
+ # Extract the version using a regular expression
109
+ match = re.search(r"__version__ = '(\S+)'", content)
110
+ if match:
111
+ # Return the version string
112
+ return match.group(1)
113
+ else:
114
+ raise RuntimeError("Could not find HIP version in the output")
115
+
116
+
117
def get_nvcc_cuda_version(cuda_dir: str) -> Version:
118
"""Get the CUDA version from nvcc.
119
@@ -210,6 +237,9 @@ elif _is_hip():
237
f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
238
f"amdgpu_arch_found: {amd_arch}")
239
240
+elif _is_neuron():
241
+ neuronxcc_version = get_neuronxcc_version()
242
+
243
ext_modules = []
244
245
vllm_extension_sources = [
@@ -227,15 +257,16 @@ vllm_extension_sources = [
257
if _is_cuda():
258
vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
259
230
-vllm_extension = CUDAExtension(
231
- name="vllm._C",
232
- sources=vllm_extension_sources,
233
- extra_compile_args={
234
- "cxx": CXX_FLAGS,
235
- "nvcc": NVCC_FLAGS,
236
- },
237
-)
238
-ext_modules.append(vllm_extension)
260
+if not _is_neuron():
261
+ vllm_extension = CUDAExtension(
262
+ name="vllm._C",
263
+ sources=vllm_extension_sources,
264
+ extra_compile_args={
265
+ "cxx": CXX_FLAGS,
266
+ "nvcc": NVCC_FLAGS,
267
+ },
268
+ )
269
+ ext_modules.append(vllm_extension)
270
271
272
def get_path(*filepath) -> str:
@@ -264,6 +295,12 @@ def get_vllm_version() -> str:
295
if hipcc_version != MAIN_CUDA_VERSION:
296
rocm_version_str = hipcc_version.replace(".", "")[:3]
297
version += f"+rocm{rocm_version_str}"
298
+ elif _is_neuron():
299
+ # Get the Neuron version
300
+ neuron_version = str(neuronxcc_version)
301
+ if neuron_version != MAIN_CUDA_VERSION:
302
+ neuron_version_str = neuron_version.replace(".", "")[:3]
303
+ version += f"+neuron{neuron_version_str}"
304
else:
305
cuda_version = str(nvcc_cuda_version)
306
if cuda_version != MAIN_CUDA_VERSION:
@@ -287,6 +324,9 @@ def get_requirements() -> List[str]:
324
if _is_hip():
325
with open(get_path("requirements-rocm.txt")) as f:
326
requirements = f.read().strip().split("\n")
327
+ elif _is_neuron():
328
+ with open(get_path("requirements-neuron.txt")) as f:
329
+ requirements = f.read().strip().split("\n")
330
else:
331
with open(get_path("requirements.txt")) as f:
332
requirements = f.read().strip().split("\n")
@@ -325,6 +365,6 @@ setuptools.setup(
365
python_requires=">=3.8",
366
install_requires=get_requirements(),
367
ext_modules=ext_modules,
328
- cmdclass={"build_ext": BuildExtension},
368
+ cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
369
package_data=package_data,
370
)
vllm/utils.py
+4
-2
@@ -8,8 +8,6 @@ from typing import List
8
import psutil
9
import torch
10
11
-from vllm._C import cuda_utils
12
-
11
12
class Device(enum.Enum):
13
GPU = enum.auto()
@@ -36,6 +34,10 @@ def is_hip() -> bool:
34
35
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
36
"""Returns the maximum shared memory per thread block in bytes."""
37
+ # NOTE: This import statement should be executed lazily since
38
+ # the Neuron-X backend does not have the `cuda_utils` module.
39
+ from vllm._C import cuda_utils
40
+
41
# https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
42
cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
43
max_shared_mem = cuda_utils.get_device_attribute(