A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Cmake based build system (#2830)
bnellnm committed
Mar 18, 2024 at 18:38 UTC
9fdf3de346836e88b310e53b50e7947974fde1d3
10 files changed
+868
-302
CMakeLists.txt
new
+279
@@ -0,0 +1,279 @@
1
+cmake_minimum_required(VERSION 3.21)
2
+
3
+project(vllm_extensions LANGUAGES CXX)
4
+
5
+message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
6
+
7
+include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
8
+
9
+#
10
+# Supported python versions. These versions will be searched in order, the
11
+# first match will be selected. These should be kept in sync with setup.py.
12
+#
13
+set(PYTHON_SUPPORTED_VERSIONS "3.8" "3.9" "3.10" "3.11")
14
+
15
+# Supported NVIDIA architectures.
16
+set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
17
+
18
+# Supported AMD GPU architectures.
19
+set(HIP_SUPPORTED_ARCHS "gfx908;gfx90a;gfx942;gfx1100")
20
+
21
+#
22
+# Supported/expected torch versions for CUDA/ROCm.
23
+#
24
+# Currently, having an incorrect pytorch version results in a warning
25
+# rather than an error.
26
+#
27
+# Note: the CUDA torch version is derived from pyproject.toml and various
28
+# requirements.txt files and should be kept consistent. The ROCm torch
29
+# versions are derived from Dockerfile.rocm
30
+#
31
+set(TORCH_SUPPORTED_VERSION_CUDA "2.1.2")
32
+set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
33
+set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
34
+
35
+#
36
+# Try to find python package with an executable that exactly matches
37
+# `VLLM_PYTHON_EXECUTABLE` and is one of the supported versions.
38
+#
39
+if (VLLM_PYTHON_EXECUTABLE)
40
+ find_python_from_executable(${VLLM_PYTHON_EXECUTABLE} "${PYTHON_SUPPORTED_VERSIONS}")
41
+else()
42
+ message(FATAL_ERROR
43
+ "Please set VLLM_PYTHON_EXECUTABLE to the path of the desired python version"
44
+ " before running cmake configure.")
45
+endif()
46
+
47
+#
48
+# Update cmake's `CMAKE_PREFIX_PATH` with torch location.
49
+#
50
+append_cmake_prefix_path("torch" "torch.utils.cmake_prefix_path")
51
+
52
+#
53
+# Import torch cmake configuration.
54
+# Torch also imports CUDA (and partially HIP) languages with some customizations,
55
+# so there is no need to do this explicitly with check_language/enable_language,
56
+# etc.
57
+#
58
+find_package(Torch REQUIRED)
59
+
60
+#
61
+# Normally `torch.utils.cpp_extension.CUDAExtension` would add
62
+# `libtorch_python.so` for linking against an extension. Torch's cmake
63
+# configuration does not include this library (presumably since the cmake
64
+# config is used for standalone C++ binaries that link against torch).
65
+# The `libtorch_python.so` library defines some of the glue code between
66
+# torch/python via pybind and is required by VLLM extensions for this
67
+# reason. So, add it by manually using `append_torchlib_if_found` from
68
+# torch's cmake setup.
69
+#
70
+append_torchlib_if_found(torch_python)
71
+
72
+#
73
+# Set up GPU language and check the torch version and warn if it isn't
74
+# what is expected.
75
+#
76
+if (NOT HIP_FOUND AND CUDA_FOUND)
77
+ set(VLLM_GPU_LANG "CUDA")
78
+
79
+ if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
80
+ message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
81
+ "expected for CUDA build, saw ${Torch_VERSION} instead.")
82
+ endif()
83
+elseif(HIP_FOUND)
84
+ set(VLLM_GPU_LANG "HIP")
85
+
86
+ # Importing torch recognizes and sets up some HIP/ROCm configuration but does
87
+ # not let cmake recognize .hip files. In order to get cmake to understand the
88
+ # .hip extension automatically, HIP must be enabled explicitly.
89
+ enable_language(HIP)
90
+
91
+ # ROCm 5.x
92
+ if (ROCM_VERSION_DEV_MAJOR EQUAL 5 AND
93
+ NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_5X})
94
+ message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_5X} "
95
+ "expected for ROCMm 5.x build, saw ${Torch_VERSION} instead.")
96
+ endif()
97
+
98
+ # ROCm 6.x
99
+ if (ROCM_VERSION_DEV_MAJOR EQUAL 6 AND
100
+ NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_6X})
101
+ message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_6X} "
102
+ "expected for ROCMm 6.x build, saw ${Torch_VERSION} instead.")
103
+ endif()
104
+else()
105
+ message(FATAL_ERROR "Can't find CUDA or HIP installation.")
106
+endif()
107
+
108
+#
109
+# Override the GPU architectures detected by cmake/torch and filter them by
110
+# the supported versions for the current language.
111
+# The final set of arches is stored in `VLLM_GPU_ARCHES`.
112
+#
113
+override_gpu_arches(VLLM_GPU_ARCHES
114
+ ${VLLM_GPU_LANG}
115
+ "${${VLLM_GPU_LANG}_SUPPORTED_ARCHS}")
116
+
117
+#
118
+# Query torch for additional GPU compilation flags for the given
119
+# `VLLM_GPU_LANG`.
120
+# The final set of arches is stored in `VLLM_GPU_FLAGS`.
121
+#
122
+get_torch_gpu_compiler_flags(VLLM_GPU_FLAGS ${VLLM_GPU_LANG})
123
+
124
+#
125
+# Set nvcc parallelism.
126
+#
127
+if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
128
+ list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
129
+endif()
130
+
131
+#
132
+# Define extension targets
133
+#
134
+
135
+#
136
+# _C extension
137
+#
138
+
139
+set(VLLM_EXT_SRC
140
+ "csrc/cache_kernels.cu"
141
+ "csrc/attention/attention_kernels.cu"
142
+ "csrc/pos_encoding_kernels.cu"
143
+ "csrc/activation_kernels.cu"
144
+ "csrc/layernorm_kernels.cu"
145
+ "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
146
+ "csrc/quantization/gptq/q_gemm.cu"
147
+ "csrc/cuda_utils_kernels.cu"
148
+ "csrc/moe_align_block_size_kernels.cu"
149
+ "csrc/pybind.cpp")
150
+
151
+if(VLLM_GPU_LANG STREQUAL "CUDA")
152
+ list(APPEND VLLM_EXT_SRC
153
+ "csrc/quantization/awq/gemm_kernels.cu"
154
+ "csrc/quantization/marlin/marlin_cuda_kernel.cu"
155
+ "csrc/custom_all_reduce.cu")
156
+endif()
157
+
158
+define_gpu_extension_target(
159
+ _C
160
+ DESTINATION vllm
161
+ LANGUAGE ${VLLM_GPU_LANG}
162
+ SOURCES ${VLLM_EXT_SRC}
163
+ COMPILE_FLAGS ${VLLM_GPU_FLAGS}
164
+ ARCHITECTURES ${VLLM_GPU_ARCHES}
165
+ WITH_SOABI)
166
+
167
+#
168
+# _moe_C extension
169
+#
170
+
171
+set(VLLM_MOE_EXT_SRC
172
+ "csrc/moe/moe_ops.cpp"
173
+ "csrc/moe/topk_softmax_kernels.cu")
174
+
175
+define_gpu_extension_target(
176
+ _moe_C
177
+ DESTINATION vllm
178
+ LANGUAGE ${VLLM_GPU_LANG}
179
+ SOURCES ${VLLM_MOE_EXT_SRC}
180
+ COMPILE_FLAGS ${VLLM_GPU_FLAGS}
181
+ ARCHITECTURES ${VLLM_GPU_ARCHES}
182
+ WITH_SOABI)
183
+
184
+#
185
+# _punica_C extension
186
+#
187
+
188
+set(VLLM_PUNICA_EXT_SRC
189
+ "csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
190
+ "csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu"
191
+ "csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu"
192
+ "csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu"
193
+ "csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
194
+ "csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu"
195
+ "csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu"
196
+ "csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu"
197
+ "csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu"
198
+ "csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
199
+ "csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu"
200
+ "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
201
+ "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
202
+ "csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu"
203
+ "csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu"
204
+ "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
205
+ "csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu"
206
+ "csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu"
207
+ "csrc/punica/punica_ops.cc")
208
+
209
+#
210
+# Copy GPU compilation flags+update for punica
211
+#
212
+set(VLLM_PUNICA_GPU_FLAGS ${VLLM_GPU_FLAGS})
213
+list(REMOVE_ITEM VLLM_PUNICA_GPU_FLAGS
214
+ "-D__CUDA_NO_HALF_OPERATORS__"
215
+ "-D__CUDA_NO_HALF_CONVERSIONS__"
216
+ "-D__CUDA_NO_BFLOAT16_CONVERSIONS__"
217
+ "-D__CUDA_NO_HALF2_OPERATORS__")
218
+
219
+#
220
+# Filter out CUDA architectures < 8.0 for punica.
221
+#
222
+if (${VLLM_GPU_LANG} STREQUAL "CUDA")
223
+ set(VLLM_PUNICA_GPU_ARCHES)
224
+ foreach(ARCH ${VLLM_GPU_ARCHES})
225
+ string_to_ver(CODE_VER ${ARCH})
226
+ if (CODE_VER GREATER_EQUAL 8.0)
227
+ list(APPEND VLLM_PUNICA_GPU_ARCHES ${ARCH})
228
+ endif()
229
+ endforeach()
230
+ message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
231
+endif()
232
+
233
+if (VLLM_PUNICA_GPU_ARCHES)
234
+ define_gpu_extension_target(
235
+ _punica_C
236
+ DESTINATION vllm
237
+ LANGUAGE ${VLLM_GPU_LANG}
238
+ SOURCES ${VLLM_PUNICA_EXT_SRC}
239
+ COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
240
+ ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
241
+ WITH_SOABI)
242
+else()
243
+ message(WARNING "Unable to create _punica_C target because none of the "
244
+ "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0")
245
+endif()
246
+
247
+#
248
+# Add the `default` target which detects which extensions should be
249
+# built based on platform/architecture. This is the same logic that
250
+# setup.py uses to select which extensions should be built and should
251
+# be kept in sync.
252
+#
253
+# The `default` target makes direct use of cmake easier since knowledge
254
+# of which extensions are supported has been factored in, e.g.
255
+#
256
+# mkdir build && cd build
257
+# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
258
+# cmake --build . --target default
259
+#
260
+add_custom_target(default)
261
+
262
+if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
263
+ message(STATUS "Enabling C extension.")
264
+ add_dependencies(default _C)
265
+endif()
266
+
267
+if(VLLM_GPU_LANG STREQUAL "CUDA")
268
+ message(STATUS "Enabling moe extension.")
269
+ add_dependencies(default _moe_C)
270
+
271
+ # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
272
+ # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
273
+ # there are supported target arches.
274
+ if (VLLM_PUNICA_GPU_ARCHES AND
275
+ (ENV{VLLM_INSTALL_PUNICA_KERNELS} OR VLLM_INSTALL_PUNICA_KERNELS))
276
+ message(STATUS "Enabling punica extension.")
277
+ add_dependencies(default _punica_C)
278
+ endif()
279
+endif()
Dockerfile
+2
@@ -38,6 +38,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
38
# copy input files
39
COPY csrc csrc
40
COPY setup.py setup.py
41
+COPY cmake cmake
42
+COPY CMakeLists.txt CMakeLists.txt
43
COPY requirements.txt requirements.txt
44
COPY pyproject.toml pyproject.toml
45
COPY vllm/__init__.py vllm/__init__.py
MANIFEST.in
+2
@@ -1,4 +1,6 @@
1
include LICENSE
2
include requirements.txt
3
+include CMakeLists.txt
4
5
+recursive-include cmake *
6
recursive-include csrc *
cmake/hipify.py
new
+73
@@ -0,0 +1,73 @@
1
+#!/usr/bin/env python3
2
+
3
+#
4
+# A command line tool for running pytorch's hipify preprocessor on CUDA
5
+# source files.
6
+#
7
+# See https://github.com/ROCm/hipify_torch
8
+# and <torch install dir>/utils/hipify/hipify_python.py
9
+#
10
+
11
+import argparse
12
+import shutil
13
+import os
14
+
15
+from torch.utils.hipify.hipify_python import hipify
16
+
17
+if __name__ == '__main__':
18
+ parser = argparse.ArgumentParser()
19
+
20
+ # Project directory where all the source + include files live.
21
+ parser.add_argument(
22
+ "-p",
23
+ "--project_dir",
24
+ help="The project directory.",
25
+ )
26
+
27
+ # Directory where hipified files are written.
28
+ parser.add_argument(
29
+ "-o",
30
+ "--output_dir",
31
+ help="The output directory.",
32
+ )
33
+
34
+ # Source files to convert.
35
+ parser.add_argument("sources",
36
+ help="Source files to hipify.",
37
+ nargs="*",
38
+ default=[])
39
+
40
+ args = parser.parse_args()
41
+
42
+ # Limit include scope to project_dir only
43
+ includes = [os.path.join(args.project_dir, '*')]
44
+
45
+ # Get absolute path for all source files.
46
+ extra_files = [os.path.abspath(s) for s in args.sources]
47
+
48
+ # Copy sources from project directory to output directory.
49
+ # The directory might already exist to hold object files so we ignore that.
50
+ shutil.copytree(args.project_dir, args.output_dir, dirs_exist_ok=True)
51
+
52
+ hipify_result = hipify(project_directory=args.project_dir,
53
+ output_directory=args.output_dir,
54
+ header_include_dirs=[],
55
+ includes=includes,
56
+ extra_files=extra_files,
57
+ show_detailed=True,
58
+ is_pytorch_extension=True,
59
+ hipify_extra_files_only=True)
60
+
61
+ hipified_sources = []
62
+ for source in args.sources:
63
+ s_abs = os.path.abspath(source)
64
+ hipified_s_abs = (hipify_result[s_abs].hipified_path if
65
+ (s_abs in hipify_result
66
+ and hipify_result[s_abs].hipified_path is not None)
67
+ else s_abs)
68
+ hipified_sources.append(hipified_s_abs)
69
+
70
+ assert (len(hipified_sources) == len(args.sources))
71
+
72
+ # Print hipified source files.
73
+ print("\n".join(hipified_sources))
cmake/utils.cmake
new
+334
@@ -0,0 +1,334 @@
1
+#
2
+# Attempt to find the python package that uses the same python executable as
3
+# `EXECUTABLE` and is one of the `SUPPORTED_VERSIONS`.
4
+#
5
+macro (find_python_from_executable EXECUTABLE SUPPORTED_VERSIONS)
6
+ file(REAL_PATH ${EXECUTABLE} EXECUTABLE)
7
+ set(Python_EXECUTABLE ${EXECUTABLE})
8
+ find_package(Python COMPONENTS Interpreter Development.Module)
9
+ if (NOT Python_FOUND)
10
+ message(FATAL_ERROR "Unable to find python matching: ${EXECUTABLE}.")
11
+ endif()
12
+ set(_VER "${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}")
13
+ set(_SUPPORTED_VERSIONS_LIST ${SUPPORTED_VERSIONS} ${ARGN})
14
+ if (NOT _VER IN_LIST _SUPPORTED_VERSIONS_LIST)
15
+ message(FATAL_ERROR
16
+ "Python version (${_VER}) is not one of the supported versions: "
17
+ "${_SUPPORTED_VERSIONS_LIST}.")
18
+ endif()
19
+ message(STATUS "Found python matching: ${EXECUTABLE}.")
20
+endmacro()
21
+
22
+#
23
+# Run `EXPR` in python. The standard output of python is stored in `OUT` and
24
+# has trailing whitespace stripped. If an error is encountered when running
25
+# python, a fatal message `ERR_MSG` is issued.
26
+#
27
+function (run_python OUT EXPR ERR_MSG)
28
+ execute_process(
29
+ COMMAND
30
+ "${Python_EXECUTABLE}" "-c" "${EXPR}"
31
+ OUTPUT_VARIABLE PYTHON_OUT
32
+ RESULT_VARIABLE PYTHON_ERROR_CODE
33
+ ERROR_VARIABLE PYTHON_STDERR
34
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
35
+
36
+ if(NOT PYTHON_ERROR_CODE EQUAL 0)
37
+ message(FATAL_ERROR "${ERR_MSG}: ${PYTHON_STDERR}")
38
+ endif()
39
+ set(${OUT} ${PYTHON_OUT} PARENT_SCOPE)
40
+endfunction()
41
+
42
+# Run `EXPR` in python after importing `PKG`. Use the result of this to extend
43
+# `CMAKE_PREFIX_PATH` so the torch cmake configuration can be imported.
44
+macro (append_cmake_prefix_path PKG EXPR)
45
+ run_python(_PREFIX_PATH
46
+ "import ${PKG}; print(${EXPR})" "Failed to locate ${PKG} path")
47
+ list(APPEND CMAKE_PREFIX_PATH ${_PREFIX_PATH})
48
+endmacro()
49
+
50
+#
51
+# Add a target named `hipify${NAME}` that runs the hipify preprocessor on a set
52
+# of CUDA source files. The names of the corresponding "hipified" sources are
53
+# stored in `OUT_SRCS`.
54
+#
55
+function (hipify_sources_target OUT_SRCS NAME ORIG_SRCS)
56
+ #
57
+ # Split into C++ and non-C++ (i.e. CUDA) sources.
58
+ #
59
+ set(SRCS ${ORIG_SRCS})
60
+ set(CXX_SRCS ${ORIG_SRCS})
61
+ list(FILTER SRCS EXCLUDE REGEX "\.(cc)|(cpp)$")
62
+ list(FILTER CXX_SRCS INCLUDE REGEX "\.(cc)|(cpp)$")
63
+
64
+ #
65
+ # Generate ROCm/HIP source file names from CUDA file names.
66
+ # Since HIP files are generated code, they will appear in the build area
67
+ # `CMAKE_CURRENT_BINARY_DIR` directory rather than the original csrc dir.
68
+ #
69
+ set(HIP_SRCS)
70
+ foreach (SRC ${SRCS})
71
+ string(REGEX REPLACE "\.cu$" "\.hip" SRC ${SRC})
72
+ string(REGEX REPLACE "cuda" "hip" SRC ${SRC})
73
+ list(APPEND HIP_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${SRC}")
74
+ endforeach()
75
+
76
+ set(CSRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/csrc)
77
+ add_custom_target(
78
+ hipify${NAME}
79
+ COMMAND ${CMAKE_SOURCE_DIR}/cmake/hipify.py -p ${CMAKE_SOURCE_DIR}/csrc -o ${CSRC_BUILD_DIR} ${SRCS}
80
+ DEPENDS ${CMAKE_SOURCE_DIR}/cmake/hipify.py ${SRCS}
81
+ BYPRODUCTS ${HIP_SRCS}
82
+ COMMENT "Running hipify on ${NAME} extension source files.")
83
+
84
+ # Swap out original extension sources with hipified sources.
85
+ list(APPEND HIP_SRCS ${CXX_SRCS})
86
+ set(${OUT_SRCS} ${HIP_SRCS} PARENT_SCOPE)
87
+endfunction()
88
+
89
+#
90
+# Get additional GPU compiler flags from torch.
91
+#
92
+function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
93
+ if (${GPU_LANG} STREQUAL "CUDA")
94
+ #
95
+ # Get common NVCC flags from torch.
96
+ #
97
+ run_python(GPU_FLAGS
98
+ "from torch.utils.cpp_extension import COMMON_NVCC_FLAGS; print(';'.join(COMMON_NVCC_FLAGS))"
99
+ "Failed to determine torch nvcc compiler flags")
100
+
101
+ if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102
+ list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
103
+ endif()
104
+
105
+ elseif(${GPU_LANG} STREQUAL "HIP")
106
+ #
107
+ # Get common HIP/HIPCC flags from torch.
108
+ #
109
+ run_python(GPU_FLAGS
110
+ "import torch.utils.cpp_extension as t; print(';'.join(t.COMMON_HIP_FLAGS + t.COMMON_HIPCC_FLAGS))"
111
+ "Failed to determine torch nvcc compiler flags")
112
+
113
+ list(APPEND GPU_FLAGS
114
+ "-DUSE_ROCM"
115
+ "-U__HIP_NO_HALF_CONVERSIONS__"
116
+ "-U__HIP_NO_HALF_OPERATORS__"
117
+ "-fno-gpu-rdc")
118
+
119
+ endif()
120
+ set(${OUT_GPU_FLAGS} ${GPU_FLAGS} PARENT_SCOPE)
121
+endfunction()
122
+
123
+# Macro for converting a `gencode` version number to a cmake version number.
124
+macro(string_to_ver OUT_VER IN_STR)
125
+ string(REGEX REPLACE "\([0-9]+\)\([0-9]\)" "\\1.\\2" ${OUT_VER} ${IN_STR})
126
+endmacro()
127
+
128
+#
129
+# Override the GPU architectures detected by cmake/torch and filter them by
130
+# `GPU_SUPPORTED_ARCHES`. Sets the final set of architectures in
131
+# `GPU_ARCHES`.
132
+#
133
+# Note: this is defined as a macro since it updates `CMAKE_CUDA_FLAGS`.
134
+#
135
+macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
136
+ set(_GPU_SUPPORTED_ARCHES_LIST ${GPU_SUPPORTED_ARCHES} ${ARGN})
137
+ message(STATUS "${GPU_LANG} supported arches: ${_GPU_SUPPORTED_ARCHES_LIST}")
138
+
139
+ if (${GPU_LANG} STREQUAL "HIP")
140
+ #
141
+ # `GPU_ARCHES` controls the `--offload-arch` flags.
142
+ # `CMAKE_HIP_ARCHITECTURES` is set up by torch and can be controlled
143
+ # via the `PYTORCH_ROCM_ARCH` env variable.
144
+ #
145
+
146
+ #
147
+ # Find the intersection of the supported + detected architectures to
148
+ # set the module architecture flags.
149
+ #
150
+ set(${GPU_ARCHES})
151
+ foreach (_ARCH ${CMAKE_HIP_ARCHITECTURES})
152
+ if (_ARCH IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
153
+ list(APPEND ${GPU_ARCHES} ${_ARCH})
154
+ endif()
155
+ endforeach()
156
+
157
+ if(NOT ${GPU_ARCHES})
158
+ message(FATAL_ERROR
159
+ "None of the detected ROCm architectures: ${CMAKE_HIP_ARCHITECTURES} is"
160
+ " supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
161
+ endif()
162
+
163
+ elseif(${GPU_LANG} STREQUAL "CUDA")
164
+ #
165
+ # Setup/process CUDA arch flags.
166
+ #
167
+ # The torch cmake setup hardcodes the detected architecture flags in
168
+ # `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
169
+ # can't modified on a per-target basis, e.g. for the `punica` extension.
170
+ # So, all the `-gencode` flags need to be extracted and removed from
171
+ # `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
172
+ # Since it's not possible to use `target_compiler_options` for adding target
173
+ # specific `-gencode` arguments, the target's `CUDA_ARCHITECTURES` property
174
+ # must be used instead. This requires repackaging the architecture flags
175
+ # into a format that cmake expects for `CUDA_ARCHITECTURES`.
176
+ #
177
+ # This is a bit fragile in that it depends on torch using `-gencode` as opposed
178
+ # to one of the other nvcc options to specify architectures.
179
+ #
180
+ # Note: torch uses the `TORCH_CUDA_ARCH_LIST` environment variable to override
181
+ # detected architectures.
182
+ #
183
+ message(DEBUG "initial CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
184
+
185
+ # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
186
+ string(REGEX MATCHALL "-gencode arch=[^ ]+" _CUDA_ARCH_FLAGS
187
+ ${CMAKE_CUDA_FLAGS})
188
+
189
+ # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
190
+ # and passed back via the `CUDA_ARCHITECTURES` property.
191
+ string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
192
+ ${CMAKE_CUDA_FLAGS})
193
+
194
+ # If this error is triggered, it might mean that torch has changed how it sets
195
+ # up nvcc architecture code generation flags.
196
+ if (NOT _CUDA_ARCH_FLAGS)
197
+ message(FATAL_ERROR
198
+ "Could not find any architecture related code generation flags in "
199
+ "CMAKE_CUDA_FLAGS. (${CMAKE_CUDA_FLAGS})")
200
+ endif()
201
+
202
+ message(DEBUG "final CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
203
+ message(DEBUG "arch flags: ${_CUDA_ARCH_FLAGS}")
204
+
205
+ # Initialize the architecture lists to empty.
206
+ set(${GPU_ARCHES})
207
+
208
+ # Process each `gencode` flag.
209
+ foreach(_ARCH ${_CUDA_ARCH_FLAGS})
210
+ # For each flag, extract the version number and whether it refers to PTX
211
+ # or native code.
212
+ # Note: if a regex matches then `CMAKE_MATCH_1` holds the binding
213
+ # for that match.
214
+
215
+ string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
216
+ if (_COMPUTE)
217
+ set(_COMPUTE ${CMAKE_MATCH_1})
218
+ endif()
219
+
220
+ string(REGEX MATCH "code=sm_\([0-9]+a?\)" _SM ${_ARCH})
221
+ if (_SM)
222
+ set(_SM ${CMAKE_MATCH_1})
223
+ endif()
224
+
225
+ string(REGEX MATCH "code=compute_\([0-9]+a?\)" _CODE ${_ARCH})
226
+ if (_CODE)
227
+ set(_CODE ${CMAKE_MATCH_1})
228
+ endif()
229
+
230
+ # Make sure the virtual architecture can be matched.
231
+ if (NOT _COMPUTE)
232
+ message(FATAL_ERROR
233
+ "Could not determine virtual architecture from: ${_ARCH}.")
234
+ endif()
235
+
236
+ # One of sm_ or compute_ must exist.
237
+ if ((NOT _SM) AND (NOT _CODE))
238
+ message(FATAL_ERROR
239
+ "Could not determine a codegen architecture from: ${_ARCH}.")
240
+ endif()
241
+
242
+ if (_SM)
243
+ set(_VIRT "")
244
+ set(_CODE_ARCH ${_SM})
245
+ else()
246
+ set(_VIRT "-virtual")
247
+ set(_CODE_ARCH ${_CODE})
248
+ endif()
249
+
250
+ # Check if the current version is in the supported arch list.
251
+ string_to_ver(_CODE_VER ${_CODE_ARCH})
252
+ if (NOT _CODE_VER IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
253
+ message(STATUS "discarding unsupported CUDA arch ${_VER}.")
254
+ continue()
255
+ endif()
256
+
257
+ # Add it to the arch list.
258
+ list(APPEND ${GPU_ARCHES} "${_CODE_ARCH}${_VIRT}")
259
+ endforeach()
260
+ endif()
261
+ message(STATUS "${GPU_LANG} target arches: ${${GPU_ARCHES}}")
262
+endmacro()
263
+
264
+#
265
+# Define a target named `GPU_MOD_NAME` for a single extension. The
266
+# arguments are:
267
+#
268
+# DESTINATION <dest> - Module destination directory.
269
+# LANGUAGE <lang> - The GPU language for this module, e.g CUDA, HIP,
270
+# etc.
271
+# SOURCES <sources> - List of source files relative to CMakeLists.txt
272
+# directory.
273
+#
274
+# Optional arguments:
275
+#
276
+# ARCHITECTURES <arches> - A list of target GPU architectures in cmake
277
+# format.
278
+# Refer `CMAKE_CUDA_ARCHITECTURES` documentation
279
+# and `CMAKE_HIP_ARCHITECTURES` for more info.
280
+# ARCHITECTURES will use cmake's defaults if
281
+# not provided.
282
+# COMPILE_FLAGS <flags> - Extra compiler flags passed to NVCC/hip.
283
+# INCLUDE_DIRECTORIES <dirs> - Extra include directories.
284
+# LINK_LIBRARIES <libraries> - Extra link libraries.
285
+# WITH_SOABI - Generate library with python SOABI suffix name.
286
+#
287
+# Note: optimization level/debug info is set via cmake build type.
288
+#
289
+function (define_gpu_extension_target GPU_MOD_NAME)
290
+ cmake_parse_arguments(PARSE_ARGV 1
291
+ GPU
292
+ "WITH_SOABI"
293
+ "DESTINATION;LANGUAGE"
294
+ "SOURCES;ARCHITECTURES;COMPILE_FLAGS;INCLUDE_DIRECTORIES;LIBRARIES")
295
+
296
+ # Add hipify preprocessing step when building with HIP/ROCm.
297
+ if (GPU_LANGUAGE STREQUAL "HIP")
298
+ hipify_sources_target(GPU_SOURCES ${GPU_MOD_NAME} "${GPU_SOURCES}")
299
+ endif()
300
+
301
+ if (GPU_WITH_SOABI)
302
+ set(GPU_WITH_SOABI WITH_SOABI)
303
+ else()
304
+ set(GPU_WITH_SOABI)
305
+ endif()
306
+
307
+ Python_add_library(${GPU_MOD_NAME} MODULE "${GPU_SOURCES}" ${GPU_WITH_SOABI})
308
+
309
+ if (GPU_LANGUAGE STREQUAL "HIP")
310
+ # Make this target dependent on the hipify preprocessor step.
311
+ add_dependencies(${GPU_MOD_NAME} hipify${GPU_MOD_NAME})
312
+ endif()
313
+
314
+ if (GPU_ARCHITECTURES)
315
+ set_target_properties(${GPU_MOD_NAME} PROPERTIES
316
+ ${GPU_LANGUAGE}_ARCHITECTURES "${GPU_ARCHITECTURES}")
317
+ endif()
318
+
319
+ set_property(TARGET ${GPU_MOD_NAME} PROPERTY CXX_STANDARD 17)
320
+
321
+ target_compile_options(${GPU_MOD_NAME} PRIVATE
322
+ $<$<COMPILE_LANGUAGE:${GPU_LANGUAGE}>:${GPU_COMPILE_FLAGS}>)
323
+
324
+ target_compile_definitions(${GPU_MOD_NAME} PRIVATE
325
+ "-DTORCH_EXTENSION_NAME=${GPU_MOD_NAME}")
326
+
327
+ target_include_directories(${GPU_MOD_NAME} PRIVATE csrc
328
+ ${GPU_INCLUDE_DIRECTORIES})
329
+
330
+ target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES}
331
+ ${GPU_LIBRARIES})
332
+
333
+ install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION})
334
+endfunction()
pyproject.toml
+1
@@ -1,6 +1,7 @@
1
[build-system]
2
# Should be mirrored in requirements-build.txt
3
requires = [
4
+ "cmake>=3.21",
5
"ninja",
6
"packaging",
7
"setuptools >= 49.4.0",
requirements-build.txt
+2
-1
@@ -1,6 +1,7 @@
1
# Should be mirrored in pyproject.toml
2
+cmake>=3.21
3
ninja
4
packaging
5
setuptools>=49.4.0
6
torch==2.1.2
6
-wheel
\ No newline at end of file
7
+wheel
requirements-rocm.txt
+1
@@ -1,3 +1,4 @@
1
+cmake>=3.21
2
ninja # For faster builds.
3
typing-extensions>=4.8.0
4
starlette
requirements.txt
+1
@@ -1,3 +1,4 @@
1
+cmake>=3.21
2
ninja # For faster builds.
3
psutil
4
ray >= 2.9
setup.py
+173
-301
@@ -1,23 +1,16 @@
1
-import contextlib
1
import io
2
import os
3
import re
4
import subprocess
6
-import warnings
7
-from pathlib import Path
8
-from typing import List, Set
5
+import sys
6
+from typing import List
7
8
from packaging.version import parse, Version
11
-import setuptools
12
-import sys
9
+from setuptools import setup, find_packages, Extension
10
+from setuptools.command.build_ext import build_ext
11
+from shutil import which
12
import torch
14
-import torch.utils.cpp_extension as torch_cpp_ext
15
-from torch.utils.cpp_extension import (
16
- BuildExtension,
17
- CUDAExtension,
18
- CUDA_HOME,
19
- ROCM_HOME,
20
-)
13
+from torch.utils.cpp_extension import CUDA_HOME
14
15
ROOT_DIR = os.path.dirname(__file__)
16
@@ -25,17 +18,153 @@ ROOT_DIR = os.path.dirname(__file__)
18
assert sys.platform.startswith(
19
"linux"), "vLLM only supports Linux platform (including WSL)."
20
28
-# If you are developing the C++ backend of vLLM, consider building vLLM with
29
-# `python setup.py develop` since it will give you incremental builds.
30
-# The downside is that this method is deprecated, see
31
-# https://github.com/pypa/setuptools/issues/917
32
-
21
MAIN_CUDA_VERSION = "12.1"
22
35
-# Supported NVIDIA GPU architectures.
36
-NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
37
-ROCM_SUPPORTED_ARCHS = {"gfx908", "gfx90a", "gfx942", "gfx1100"}
38
-# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
23
+
24
+def is_sccache_available() -> bool:
25
+ return which("sccache") is not None
26
+
27
+
28
+def is_ccache_available() -> bool:
29
+ return which("ccache") is not None
30
+
31
+
32
+def is_ninja_available() -> bool:
33
+ return which("ninja") is not None
34
+
35
+
36
+def remove_prefix(text, prefix):
37
+ if text.startswith(prefix):
38
+ return text[len(prefix):]
39
+ return text
40
+
41
+
42
+class CMakeExtension(Extension):
43
+
44
+ def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
45
+ super().__init__(name, sources=[], **kwa)
46
+ self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
47
+
48
+
49
+class cmake_build_ext(build_ext):
50
+ # A dict of extension directories that have been configured.
51
+ did_config = {}
52
+
53
+ #
54
+ # Determine number of compilation jobs and optionally nvcc compile threads.
55
+ #
56
+ def compute_num_jobs(self):
57
+ try:
58
+ # os.sched_getaffinity() isn't universally available, so fall back
59
+ # to os.cpu_count() if we get an error here.
60
+ num_jobs = len(os.sched_getaffinity(0))
61
+ except AttributeError:
62
+ num_jobs = os.cpu_count()
63
+
64
+ nvcc_cuda_version = get_nvcc_cuda_version()
65
+ if nvcc_cuda_version >= Version("11.2"):
66
+ nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
67
+ num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
68
+ else:
69
+ nvcc_threads = None
70
+
71
+ return num_jobs, nvcc_threads
72
+
73
+ #
74
+ # Perform cmake configuration for a single extension.
75
+ #
76
+ def configure(self, ext: CMakeExtension) -> None:
77
+ # If we've already configured using the CMakeLists.txt for
78
+ # this extension, exit early.
79
+ if ext.cmake_lists_dir in cmake_build_ext.did_config:
80
+ return
81
+
82
+ cmake_build_ext.did_config[ext.cmake_lists_dir] = True
83
+
84
+ # Select the build type.
85
+ # Note: optimization level + debug info are set by the build type
86
+ default_cfg = "Debug" if self.debug else "RelWithDebInfo"
87
+ cfg = os.getenv("CMAKE_BUILD_TYPE", default_cfg)
88
+
89
+ # where .so files will be written, should be the same for all extensions
90
+ # that use the same CMakeLists.txt.
91
+ outdir = os.path.abspath(
92
+ os.path.dirname(self.get_ext_fullpath(ext.name)))
93
+
94
+ cmake_args = [
95
+ '-DCMAKE_BUILD_TYPE={}'.format(cfg),
96
+ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
97
+ '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
98
+ ]
99
+
100
+ verbose = bool(int(os.getenv('VERBOSE', '0')))
101
+ if verbose:
102
+ cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']
103
+
104
+ if is_sccache_available():
105
+ cmake_args += [
106
+ '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
107
+ '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
108
+ ]
109
+ elif is_ccache_available():
110
+ cmake_args += [
111
+ '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
112
+ '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
113
+ ]
114
+
115
+ # Pass the python executable to cmake so it can find an exact
116
+ # match.
117
+ cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]
118
+
119
+ if _install_punica():
120
+ cmake_args += ['-DVLLM_INSTALL_PUNICA_KERNELS=ON']
121
+
122
+ #
123
+ # Setup parallelism and build tool
124
+ #
125
+ num_jobs, nvcc_threads = self.compute_num_jobs()
126
+
127
+ if nvcc_threads:
128
+ cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]
129
+
130
+ if is_ninja_available():
131
+ build_tool = ['-G', 'Ninja']
132
+ cmake_args += [
133
+ '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
134
+ '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
135
+ ]
136
+ else:
137
+ # Default build tool to whatever cmake picks.
138
+ build_tool = []
139
+
140
+ subprocess.check_call(
141
+ ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
142
+ cwd=self.build_temp)
143
+
144
+ def build_extensions(self) -> None:
145
+ # Ensure that CMake is present and working
146
+ try:
147
+ subprocess.check_output(['cmake', '--version'])
148
+ except OSError as e:
149
+ raise RuntimeError('Cannot find CMake executable') from e
150
+
151
+ # Create build directory if it does not exist.
152
+ if not os.path.exists(self.build_temp):
153
+ os.makedirs(self.build_temp)
154
+
155
+ # Build all the extensions
156
+ for ext in self.extensions:
157
+ self.configure(ext)
158
+
159
+ ext_target_name = remove_prefix(ext.name, "vllm.")
160
+ num_jobs, _ = self.compute_num_jobs()
161
+
162
+ build_args = [
163
+ '--build', '.', '--target', ext_target_name, '-j',
164
+ str(num_jobs)
165
+ ]
166
+
167
+ subprocess.check_call(['cmake', *build_args], cwd=self.build_temp)
168
169
170
def _is_cuda() -> bool:
@@ -55,26 +184,8 @@ def _is_neuron() -> bool:
184
return torch_neuronx_installed
185
186
58
-# Compiler flags.
59
-CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
60
-# TODO(woosuk): Should we use -O3?
61
-NVCC_FLAGS = ["-O2", "-std=c++17"]
62
-
63
-if _is_hip():
64
- if ROCM_HOME is None:
65
- raise RuntimeError("Cannot find ROCM_HOME. "
66
- "ROCm must be available to build the package.")
67
- NVCC_FLAGS += ["-DUSE_ROCM"]
68
- NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
69
- NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
70
-
71
-if _is_cuda() and CUDA_HOME is None:
72
- raise RuntimeError(
73
- "Cannot find CUDA_HOME. CUDA must be available to build the package.")
74
-
75
-ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
76
-CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
77
-NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
187
+def _install_punica() -> bool:
188
+ return bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
189
190
191
def get_hipcc_rocm_version():
@@ -99,11 +210,6 @@ def get_hipcc_rocm_version():
210
return None
211
212
102
-def glob(pattern: str):
103
- root = Path(__name__).parent
104
- return [str(p) for p in root.glob(pattern)]
105
-
106
-
213
def get_neuronxcc_version():
214
import sysconfig
215
site_dir = sysconfig.get_paths()["purelib"]
@@ -123,12 +229,12 @@ def get_neuronxcc_version():
229
raise RuntimeError("Could not find HIP version in the output")
230
231
126
-def get_nvcc_cuda_version(cuda_dir: str) -> Version:
232
+def get_nvcc_cuda_version() -> Version:
233
"""Get the CUDA version from nvcc.
234
235
Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
236
"""
131
- nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
237
+ nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
238
universal_newlines=True)
239
output = nvcc_output.split()
240
release_idx = output.index("release") + 1
@@ -136,250 +242,6 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
242
return nvcc_cuda_version
243
244
139
-def get_pytorch_rocm_arch() -> Set[str]:
140
- """Get the cross section of Pytorch,and vllm supported gfx arches
141
-
142
- ROCM can get the supported gfx architectures in one of two ways
143
- Either through the PYTORCH_ROCM_ARCH env var, or output from
144
- rocm_agent_enumerator.
145
-
146
- In either case we can generate a list of supported arch's and
147
- cross reference with VLLM's own ROCM_SUPPORTED_ARCHs.
148
- """
149
- env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
150
-
151
- # If we don't have PYTORCH_ROCM_ARCH specified pull the list from
152
- # rocm_agent_enumerator
153
- if env_arch_list is None:
154
- command = "rocm_agent_enumerator"
155
- env_arch_list = (subprocess.check_output(
156
- [command]).decode('utf-8').strip().replace("\n", ";"))
157
- arch_source_str = "rocm_agent_enumerator"
158
- else:
159
- arch_source_str = "PYTORCH_ROCM_ARCH env variable"
160
-
161
- # List are separated by ; or space.
162
- pytorch_rocm_arch = set(env_arch_list.replace(" ", ";").split(";"))
163
-
164
- # Filter out the invalid architectures and print a warning.
165
- arch_list = pytorch_rocm_arch.intersection(ROCM_SUPPORTED_ARCHS)
166
-
167
- # If none of the specified architectures are valid, raise an error.
168
- if not arch_list:
169
- raise RuntimeError(
170
- f"None of the ROCM architectures in {arch_source_str} "
171
- f"({env_arch_list}) is supported. "
172
- f"Supported ROCM architectures are: {ROCM_SUPPORTED_ARCHS}.")
173
- invalid_arch_list = pytorch_rocm_arch - ROCM_SUPPORTED_ARCHS
174
- if invalid_arch_list:
175
- warnings.warn(
176
- f"Unsupported ROCM architectures ({invalid_arch_list}) are "
177
- f"excluded from the {arch_source_str} output "
178
- f"({env_arch_list}). Supported ROCM architectures are: "
179
- f"{ROCM_SUPPORTED_ARCHS}.",
180
- stacklevel=2)
181
- return arch_list
182
-
183
-
184
-def get_torch_arch_list() -> Set[str]:
185
- # TORCH_CUDA_ARCH_LIST can have one or more architectures,
186
- # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
187
- # compiler to additionally include PTX code that can be runtime-compiled
188
- # and executed on the 8.6 or newer architectures. While the PTX code will
189
- # not give the best performance on the newer architectures, it provides
190
- # forward compatibility.
191
- env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
192
- if env_arch_list is None:
193
- return set()
194
-
195
- # List are separated by ; or space.
196
- torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
197
- if not torch_arch_list:
198
- return set()
199
-
200
- # Filter out the invalid architectures and print a warning.
201
- valid_archs = NVIDIA_SUPPORTED_ARCHS.union(
202
- {s + "+PTX"
203
- for s in NVIDIA_SUPPORTED_ARCHS})
204
- arch_list = torch_arch_list.intersection(valid_archs)
205
- # If none of the specified architectures are valid, raise an error.
206
- if not arch_list:
207
- raise RuntimeError(
208
- "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
209
- f"variable ({env_arch_list}) is supported. "
210
- f"Supported CUDA architectures are: {valid_archs}.")
211
- invalid_arch_list = torch_arch_list - valid_archs
212
- if invalid_arch_list:
213
- warnings.warn(
214
- f"Unsupported CUDA architectures ({invalid_arch_list}) are "
215
- "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
216
- f"({env_arch_list}). Supported CUDA architectures are: "
217
- f"{valid_archs}.",
218
- stacklevel=2)
219
- return arch_list
220
-
221
-
222
-if _is_hip():
223
- rocm_arches = get_pytorch_rocm_arch()
224
- NVCC_FLAGS += ["--offload-arch=" + arch for arch in rocm_arches]
225
-else:
226
- # First, check the TORCH_CUDA_ARCH_LIST environment variable.
227
- compute_capabilities = get_torch_arch_list()
228
-
229
-if _is_cuda() and not compute_capabilities:
230
- # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
231
- # GPUs on the current machine.
232
- device_count = torch.cuda.device_count()
233
- for i in range(device_count):
234
- major, minor = torch.cuda.get_device_capability(i)
235
- if major < 7:
236
- raise RuntimeError(
237
- "GPUs with compute capability below 7.0 are not supported.")
238
- compute_capabilities.add(f"{major}.{minor}")
239
-
240
-ext_modules = []
241
-
242
-if _is_cuda():
243
- nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
244
- if not compute_capabilities:
245
- # If no GPU is specified nor available, add all supported architectures
246
- # based on the NVCC CUDA version.
247
- compute_capabilities = NVIDIA_SUPPORTED_ARCHS.copy()
248
- if nvcc_cuda_version < Version("11.1"):
249
- compute_capabilities.remove("8.6")
250
- if nvcc_cuda_version < Version("11.8"):
251
- compute_capabilities.remove("8.9")
252
- compute_capabilities.remove("9.0")
253
- # Validate the NVCC CUDA version.
254
- if nvcc_cuda_version < Version("11.0"):
255
- raise RuntimeError(
256
- "CUDA 11.0 or higher is required to build the package.")
257
- if (nvcc_cuda_version < Version("11.1")
258
- and any(cc.startswith("8.6") for cc in compute_capabilities)):
259
- raise RuntimeError(
260
- "CUDA 11.1 or higher is required for compute capability 8.6.")
261
- if nvcc_cuda_version < Version("11.8"):
262
- if any(cc.startswith("8.9") for cc in compute_capabilities):
263
- # CUDA 11.8 is required to generate the code targeting compute
264
- # capability 8.9. However, GPUs with compute capability 8.9 can
265
- # also run the code generated by the previous versions of CUDA 11
266
- # and targeting compute capability 8.0. Therefore, if CUDA 11.8
267
- # is not available, we target compute capability 8.0 instead of 8.9.
268
- warnings.warn(
269
- "CUDA 11.8 or higher is required for compute capability 8.9. "
270
- "Targeting compute capability 8.0 instead.",
271
- stacklevel=2)
272
- compute_capabilities = set(cc for cc in compute_capabilities
273
- if not cc.startswith("8.9"))
274
- compute_capabilities.add("8.0+PTX")
275
- if any(cc.startswith("9.0") for cc in compute_capabilities):
276
- raise RuntimeError(
277
- "CUDA 11.8 or higher is required for compute capability 9.0.")
278
-
279
- NVCC_FLAGS_PUNICA = NVCC_FLAGS.copy()
280
-
281
- # Add target compute capabilities to NVCC flags.
282
- for capability in compute_capabilities:
283
- num = capability[0] + capability[2]
284
- NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
285
- if capability.endswith("+PTX"):
286
- NVCC_FLAGS += [
287
- "-gencode", f"arch=compute_{num},code=compute_{num}"
288
- ]
289
- if int(capability[0]) >= 8:
290
- NVCC_FLAGS_PUNICA += [
291
- "-gencode", f"arch=compute_{num},code=sm_{num}"
292
- ]
293
- if capability.endswith("+PTX"):
294
- NVCC_FLAGS_PUNICA += [
295
- "-gencode", f"arch=compute_{num},code=compute_{num}"
296
- ]
297
-
298
- # Use NVCC threads to parallelize the build.
299
- if nvcc_cuda_version >= Version("11.2"):
300
- nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
301
- num_threads = min(os.cpu_count(), nvcc_threads)
302
- NVCC_FLAGS += ["--threads", str(num_threads)]
303
-
304
- if nvcc_cuda_version >= Version("11.8"):
305
- NVCC_FLAGS += ["-DENABLE_FP8_E5M2"]
306
-
307
- # changes for punica kernels
308
- NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
309
- REMOVE_NVCC_FLAGS = [
310
- '-D__CUDA_NO_HALF_OPERATORS__',
311
- '-D__CUDA_NO_HALF_CONVERSIONS__',
312
- '-D__CUDA_NO_BFLOAT16_CONVERSIONS__',
313
- '-D__CUDA_NO_HALF2_OPERATORS__',
314
- ]
315
- for flag in REMOVE_NVCC_FLAGS:
316
- with contextlib.suppress(ValueError):
317
- torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
318
-
319
- install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
320
- device_count = torch.cuda.device_count()
321
- for i in range(device_count):
322
- major, minor = torch.cuda.get_device_capability(i)
323
- if major < 8:
324
- install_punica = False
325
- break
326
- if install_punica:
327
- ext_modules.append(
328
- CUDAExtension(
329
- name="vllm._punica_C",
330
- sources=["csrc/punica/punica_ops.cc"] +
331
- glob("csrc/punica/bgmv/*.cu"),
332
- extra_compile_args={
333
- "cxx": CXX_FLAGS,
334
- "nvcc": NVCC_FLAGS_PUNICA,
335
- },
336
- ))
337
-elif _is_neuron():
338
- neuronxcc_version = get_neuronxcc_version()
339
-
340
-vllm_extension_sources = [
341
- "csrc/cache_kernels.cu",
342
- "csrc/attention/attention_kernels.cu",
343
- "csrc/pos_encoding_kernels.cu",
344
- "csrc/activation_kernels.cu",
345
- "csrc/layernorm_kernels.cu",
346
- "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
347
- "csrc/quantization/gptq/q_gemm.cu",
348
- "csrc/cuda_utils_kernels.cu",
349
- "csrc/moe_align_block_size_kernels.cu",
350
- "csrc/pybind.cpp",
351
-]
352
-
353
-if _is_cuda():
354
- vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
355
- vllm_extension_sources.append(
356
- "csrc/quantization/marlin/marlin_cuda_kernel.cu")
357
- vllm_extension_sources.append("csrc/custom_all_reduce.cu")
358
-
359
- # Add MoE kernels.
360
- ext_modules.append(
361
- CUDAExtension(
362
- name="vllm._moe_C",
363
- sources=glob("csrc/moe/*.cu") + glob("csrc/moe/*.cpp"),
364
- extra_compile_args={
365
- "cxx": CXX_FLAGS,
366
- "nvcc": NVCC_FLAGS,
367
- },
368
- ))
369
-
370
-if not _is_neuron():
371
- vllm_extension = CUDAExtension(
372
- name="vllm._C",
373
- sources=vllm_extension_sources,
374
- extra_compile_args={
375
- "cxx": CXX_FLAGS,
376
- "nvcc": NVCC_FLAGS,
377
- },
378
- libraries=["cuda"] if _is_cuda() else [],
379
- )
380
- ext_modules.append(vllm_extension)
381
-
382
-
245
def get_path(*filepath) -> str:
246
return os.path.join(ROOT_DIR, *filepath)
247
@@ -401,7 +263,7 @@ def get_vllm_version() -> str:
263
version = find_version(get_path("vllm", "__init__.py"))
264
265
if _is_cuda():
404
- cuda_version = str(nvcc_cuda_version)
266
+ cuda_version = str(get_nvcc_cuda_version())
267
if cuda_version != MAIN_CUDA_VERSION:
268
cuda_version_str = cuda_version.replace(".", "")[:3]
269
version += f"+cu{cuda_version_str}"
@@ -413,7 +275,7 @@ def get_vllm_version() -> str:
275
version += f"+rocm{rocm_version_str}"
276
elif _is_neuron():
277
# Get the Neuron version
416
- neuron_version = str(neuronxcc_version)
278
+ neuron_version = str(get_neuronxcc_version())
279
if neuron_version != MAIN_CUDA_VERSION:
280
neuron_version_str = neuron_version.replace(".", "")[:3]
281
version += f"+neuron{neuron_version_str}"
@@ -437,7 +299,7 @@ def get_requirements() -> List[str]:
299
if _is_cuda():
300
with open(get_path("requirements.txt")) as f:
301
requirements = f.read().strip().split("\n")
440
- if nvcc_cuda_version <= Version("11.8"):
302
+ if get_nvcc_cuda_version() <= Version("11.8"):
303
# replace cupy-cuda12x with cupy-cuda11x for cuda 11.x
304
for i in range(len(requirements)):
305
if requirements[i].startswith("cupy-cuda12x"):
@@ -456,14 +318,24 @@ def get_requirements() -> List[str]:
318
return requirements
319
320
321
+ext_modules = []
322
+
323
+if _is_cuda():
324
+ ext_modules.append(CMakeExtension(name="vllm._moe_C"))
325
+
326
+ if _install_punica():
327
+ ext_modules.append(CMakeExtension(name="vllm._punica_C"))
328
+
329
+if not _is_neuron():
330
+ ext_modules.append(CMakeExtension(name="vllm._C"))
331
+
332
package_data = {
333
"vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
334
}
335
if os.environ.get("VLLM_USE_PRECOMPILED"):
463
- ext_modules = []
336
package_data["vllm"].append("*.so")
337
466
-setuptools.setup(
338
+setup(
339
name="vllm",
340
version=get_vllm_version(),
341
author="vLLM Team",
@@ -485,11 +357,11 @@ setuptools.setup(
357
"License :: OSI Approved :: Apache Software License",
358
"Topic :: Scientific/Engineering :: Artificial Intelligence",
359
],
488
- packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
489
- "examples", "tests")),
360
+ packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
361
+ "tests")),
362
python_requires=">=3.8",
363
install_requires=get_requirements(),
364
ext_modules=ext_modules,
493
- cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
365
+ cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
366
package_data=package_data,
367
)