@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9fdf3de346
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Cmake based build system (#2830)

bnellnm committed Mar 18, 2024 at 18:38 UTC 9fdf3de346836e88b310e53b50e7947974fde1d3
10 files changed +868 -302
CMakeLists.txt new
+279
@@ -0,0 +1,279 @@
1 +cmake_minimum_required(VERSION 3.21)
2 +
3 +project(vllm_extensions LANGUAGES CXX)
4 +
5 +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
6 +
7 +include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
8 +
9 +#
10 +# Supported python versions. These versions will be searched in order, the
11 +# first match will be selected. These should be kept in sync with setup.py.
12 +#
13 +set(PYTHON_SUPPORTED_VERSIONS "3.8" "3.9" "3.10" "3.11")
14 +
15 +# Supported NVIDIA architectures.
16 +set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
17 +
18 +# Supported AMD GPU architectures.
19 +set(HIP_SUPPORTED_ARCHS "gfx908;gfx90a;gfx942;gfx1100")
20 +
21 +#
22 +# Supported/expected torch versions for CUDA/ROCm.
23 +#
24 +# Currently, having an incorrect pytorch version results in a warning
25 +# rather than an error.
26 +#
27 +# Note: the CUDA torch version is derived from pyproject.toml and various
28 +# requirements.txt files and should be kept consistent. The ROCm torch
29 +# versions are derived from Dockerfile.rocm
30 +#
31 +set(TORCH_SUPPORTED_VERSION_CUDA "2.1.2")
32 +set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
33 +set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
34 +
35 +#
36 +# Try to find python package with an executable that exactly matches
37 +# `VLLM_PYTHON_EXECUTABLE` and is one of the supported versions.
38 +#
39 +if (VLLM_PYTHON_EXECUTABLE)
40 + find_python_from_executable(${VLLM_PYTHON_EXECUTABLE} "${PYTHON_SUPPORTED_VERSIONS}")
41 +else()
42 + message(FATAL_ERROR
43 + "Please set VLLM_PYTHON_EXECUTABLE to the path of the desired python version"
44 + " before running cmake configure.")
45 +endif()
46 +
47 +#
48 +# Update cmake's `CMAKE_PREFIX_PATH` with torch location.
49 +#
50 +append_cmake_prefix_path("torch" "torch.utils.cmake_prefix_path")
51 +
52 +#
53 +# Import torch cmake configuration.
54 +# Torch also imports CUDA (and partially HIP) languages with some customizations,
55 +# so there is no need to do this explicitly with check_language/enable_language,
56 +# etc.
57 +#
58 +find_package(Torch REQUIRED)
59 +
60 +#
61 +# Normally `torch.utils.cpp_extension.CUDAExtension` would add
62 +# `libtorch_python.so` for linking against an extension. Torch's cmake
63 +# configuration does not include this library (presumably since the cmake
64 +# config is used for standalone C++ binaries that link against torch).
65 +# The `libtorch_python.so` library defines some of the glue code between
66 +# torch/python via pybind and is required by VLLM extensions for this
67 +# reason. So, add it by manually using `append_torchlib_if_found` from
68 +# torch's cmake setup.
69 +#
70 +append_torchlib_if_found(torch_python)
71 +
72 +#
73 +# Set up GPU language and check the torch version and warn if it isn't
74 +# what is expected.
75 +#
76 +if (NOT HIP_FOUND AND CUDA_FOUND)
77 + set(VLLM_GPU_LANG "CUDA")
78 +
79 + if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
80 + message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
81 + "expected for CUDA build, saw ${Torch_VERSION} instead.")
82 + endif()
83 +elseif(HIP_FOUND)
84 + set(VLLM_GPU_LANG "HIP")
85 +
86 + # Importing torch recognizes and sets up some HIP/ROCm configuration but does
87 + # not let cmake recognize .hip files. In order to get cmake to understand the
88 + # .hip extension automatically, HIP must be enabled explicitly.
89 + enable_language(HIP)
90 +
91 + # ROCm 5.x
92 + if (ROCM_VERSION_DEV_MAJOR EQUAL 5 AND
93 + NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_5X})
94 + message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_5X} "
95 + "expected for ROCMm 5.x build, saw ${Torch_VERSION} instead.")
96 + endif()
97 +
98 + # ROCm 6.x
99 + if (ROCM_VERSION_DEV_MAJOR EQUAL 6 AND
100 + NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_6X})
101 + message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_6X} "
102 + "expected for ROCMm 6.x build, saw ${Torch_VERSION} instead.")
103 + endif()
104 +else()
105 + message(FATAL_ERROR "Can't find CUDA or HIP installation.")
106 +endif()
107 +
108 +#
109 +# Override the GPU architectures detected by cmake/torch and filter them by
110 +# the supported versions for the current language.
111 +# The final set of arches is stored in `VLLM_GPU_ARCHES`.
112 +#
113 +override_gpu_arches(VLLM_GPU_ARCHES
114 + ${VLLM_GPU_LANG}
115 + "${${VLLM_GPU_LANG}_SUPPORTED_ARCHS}")
116 +
117 +#
118 +# Query torch for additional GPU compilation flags for the given
119 +# `VLLM_GPU_LANG`.
120 +# The final set of arches is stored in `VLLM_GPU_FLAGS`.
121 +#
122 +get_torch_gpu_compiler_flags(VLLM_GPU_FLAGS ${VLLM_GPU_LANG})
123 +
124 +#
125 +# Set nvcc parallelism.
126 +#
127 +if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
128 + list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
129 +endif()
130 +
131 +#
132 +# Define extension targets
133 +#
134 +
135 +#
136 +# _C extension
137 +#
138 +
139 +set(VLLM_EXT_SRC
140 + "csrc/cache_kernels.cu"
141 + "csrc/attention/attention_kernels.cu"
142 + "csrc/pos_encoding_kernels.cu"
143 + "csrc/activation_kernels.cu"
144 + "csrc/layernorm_kernels.cu"
145 + "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
146 + "csrc/quantization/gptq/q_gemm.cu"
147 + "csrc/cuda_utils_kernels.cu"
148 + "csrc/moe_align_block_size_kernels.cu"
149 + "csrc/pybind.cpp")
150 +
151 +if(VLLM_GPU_LANG STREQUAL "CUDA")
152 + list(APPEND VLLM_EXT_SRC
153 + "csrc/quantization/awq/gemm_kernels.cu"
154 + "csrc/quantization/marlin/marlin_cuda_kernel.cu"
155 + "csrc/custom_all_reduce.cu")
156 +endif()
157 +
158 +define_gpu_extension_target(
159 + _C
160 + DESTINATION vllm
161 + LANGUAGE ${VLLM_GPU_LANG}
162 + SOURCES ${VLLM_EXT_SRC}
163 + COMPILE_FLAGS ${VLLM_GPU_FLAGS}
164 + ARCHITECTURES ${VLLM_GPU_ARCHES}
165 + WITH_SOABI)
166 +
167 +#
168 +# _moe_C extension
169 +#
170 +
171 +set(VLLM_MOE_EXT_SRC
172 + "csrc/moe/moe_ops.cpp"
173 + "csrc/moe/topk_softmax_kernels.cu")
174 +
175 +define_gpu_extension_target(
176 + _moe_C
177 + DESTINATION vllm
178 + LANGUAGE ${VLLM_GPU_LANG}
179 + SOURCES ${VLLM_MOE_EXT_SRC}
180 + COMPILE_FLAGS ${VLLM_GPU_FLAGS}
181 + ARCHITECTURES ${VLLM_GPU_ARCHES}
182 + WITH_SOABI)
183 +
184 +#
185 +# _punica_C extension
186 +#
187 +
188 +set(VLLM_PUNICA_EXT_SRC
189 + "csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
190 + "csrc/punica/bgmv/bgmv_bf16_bf16_fp16.cu"
191 + "csrc/punica/bgmv/bgmv_bf16_fp16_bf16.cu"
192 + "csrc/punica/bgmv/bgmv_bf16_fp16_fp16.cu"
193 + "csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
194 + "csrc/punica/bgmv/bgmv_bf16_fp32_fp16.cu"
195 + "csrc/punica/bgmv/bgmv_fp16_bf16_bf16.cu"
196 + "csrc/punica/bgmv/bgmv_fp16_bf16_fp16.cu"
197 + "csrc/punica/bgmv/bgmv_fp16_fp16_bf16.cu"
198 + "csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
199 + "csrc/punica/bgmv/bgmv_fp16_fp32_bf16.cu"
200 + "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
201 + "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
202 + "csrc/punica/bgmv/bgmv_fp32_bf16_fp16.cu"
203 + "csrc/punica/bgmv/bgmv_fp32_fp16_bf16.cu"
204 + "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
205 + "csrc/punica/bgmv/bgmv_fp32_fp32_bf16.cu"
206 + "csrc/punica/bgmv/bgmv_fp32_fp32_fp16.cu"
207 + "csrc/punica/punica_ops.cc")
208 +
209 +#
210 +# Copy GPU compilation flags+update for punica
211 +#
212 +set(VLLM_PUNICA_GPU_FLAGS ${VLLM_GPU_FLAGS})
213 +list(REMOVE_ITEM VLLM_PUNICA_GPU_FLAGS
214 + "-D__CUDA_NO_HALF_OPERATORS__"
215 + "-D__CUDA_NO_HALF_CONVERSIONS__"
216 + "-D__CUDA_NO_BFLOAT16_CONVERSIONS__"
217 + "-D__CUDA_NO_HALF2_OPERATORS__")
218 +
219 +#
220 +# Filter out CUDA architectures < 8.0 for punica.
221 +#
222 +if (${VLLM_GPU_LANG} STREQUAL "CUDA")
223 + set(VLLM_PUNICA_GPU_ARCHES)
224 + foreach(ARCH ${VLLM_GPU_ARCHES})
225 + string_to_ver(CODE_VER ${ARCH})
226 + if (CODE_VER GREATER_EQUAL 8.0)
227 + list(APPEND VLLM_PUNICA_GPU_ARCHES ${ARCH})
228 + endif()
229 + endforeach()
230 + message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
231 +endif()
232 +
233 +if (VLLM_PUNICA_GPU_ARCHES)
234 + define_gpu_extension_target(
235 + _punica_C
236 + DESTINATION vllm
237 + LANGUAGE ${VLLM_GPU_LANG}
238 + SOURCES ${VLLM_PUNICA_EXT_SRC}
239 + COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
240 + ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
241 + WITH_SOABI)
242 +else()
243 + message(WARNING "Unable to create _punica_C target because none of the "
244 + "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0")
245 +endif()
246 +
247 +#
248 +# Add the `default` target which detects which extensions should be
249 +# built based on platform/architecture. This is the same logic that
250 +# setup.py uses to select which extensions should be built and should
251 +# be kept in sync.
252 +#
253 +# The `default` target makes direct use of cmake easier since knowledge
254 +# of which extensions are supported has been factored in, e.g.
255 +#
256 +# mkdir build && cd build
257 +# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
258 +# cmake --build . --target default
259 +#
260 +add_custom_target(default)
261 +
262 +if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
263 + message(STATUS "Enabling C extension.")
264 + add_dependencies(default _C)
265 +endif()
266 +
267 +if(VLLM_GPU_LANG STREQUAL "CUDA")
268 + message(STATUS "Enabling moe extension.")
269 + add_dependencies(default _moe_C)
270 +
271 + # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
272 + # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
273 + # there are supported target arches.
274 + if (VLLM_PUNICA_GPU_ARCHES AND
275 + (ENV{VLLM_INSTALL_PUNICA_KERNELS} OR VLLM_INSTALL_PUNICA_KERNELS))
276 + message(STATUS "Enabling punica extension.")
277 + add_dependencies(default _punica_C)
278 + endif()
279 +endif()
Dockerfile
+2
@@ -38,6 +38,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
38 # copy input files
39 COPY csrc csrc
40 COPY setup.py setup.py
41 +COPY cmake cmake
42 +COPY CMakeLists.txt CMakeLists.txt
43 COPY requirements.txt requirements.txt
44 COPY pyproject.toml pyproject.toml
45 COPY vllm/__init__.py vllm/__init__.py
MANIFEST.in
+2
@@ -1,4 +1,6 @@
1 include LICENSE
2 include requirements.txt
3 +include CMakeLists.txt
4
5 +recursive-include cmake *
6 recursive-include csrc *
cmake/hipify.py new
+73
@@ -0,0 +1,73 @@
1 +#!/usr/bin/env python3
2 +
3 +#
4 +# A command line tool for running pytorch's hipify preprocessor on CUDA
5 +# source files.
6 +#
7 +# See https://github.com/ROCm/hipify_torch
8 +# and <torch install dir>/utils/hipify/hipify_python.py
9 +#
10 +
11 +import argparse
12 +import shutil
13 +import os
14 +
15 +from torch.utils.hipify.hipify_python import hipify
16 +
17 +if __name__ == '__main__':
18 + parser = argparse.ArgumentParser()
19 +
20 + # Project directory where all the source + include files live.
21 + parser.add_argument(
22 + "-p",
23 + "--project_dir",
24 + help="The project directory.",
25 + )
26 +
27 + # Directory where hipified files are written.
28 + parser.add_argument(
29 + "-o",
30 + "--output_dir",
31 + help="The output directory.",
32 + )
33 +
34 + # Source files to convert.
35 + parser.add_argument("sources",
36 + help="Source files to hipify.",
37 + nargs="*",
38 + default=[])
39 +
40 + args = parser.parse_args()
41 +
42 + # Limit include scope to project_dir only
43 + includes = [os.path.join(args.project_dir, '*')]
44 +
45 + # Get absolute path for all source files.
46 + extra_files = [os.path.abspath(s) for s in args.sources]
47 +
48 + # Copy sources from project directory to output directory.
49 + # The directory might already exist to hold object files so we ignore that.
50 + shutil.copytree(args.project_dir, args.output_dir, dirs_exist_ok=True)
51 +
52 + hipify_result = hipify(project_directory=args.project_dir,
53 + output_directory=args.output_dir,
54 + header_include_dirs=[],
55 + includes=includes,
56 + extra_files=extra_files,
57 + show_detailed=True,
58 + is_pytorch_extension=True,
59 + hipify_extra_files_only=True)
60 +
61 + hipified_sources = []
62 + for source in args.sources:
63 + s_abs = os.path.abspath(source)
64 + hipified_s_abs = (hipify_result[s_abs].hipified_path if
65 + (s_abs in hipify_result
66 + and hipify_result[s_abs].hipified_path is not None)
67 + else s_abs)
68 + hipified_sources.append(hipified_s_abs)
69 +
70 + assert (len(hipified_sources) == len(args.sources))
71 +
72 + # Print hipified source files.
73 + print("\n".join(hipified_sources))
cmake/utils.cmake new
+334
@@ -0,0 +1,334 @@
1 +#
2 +# Attempt to find the python package that uses the same python executable as
3 +# `EXECUTABLE` and is one of the `SUPPORTED_VERSIONS`.
4 +#
5 +macro (find_python_from_executable EXECUTABLE SUPPORTED_VERSIONS)
6 + file(REAL_PATH ${EXECUTABLE} EXECUTABLE)
7 + set(Python_EXECUTABLE ${EXECUTABLE})
8 + find_package(Python COMPONENTS Interpreter Development.Module)
9 + if (NOT Python_FOUND)
10 + message(FATAL_ERROR "Unable to find python matching: ${EXECUTABLE}.")
11 + endif()
12 + set(_VER "${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}")
13 + set(_SUPPORTED_VERSIONS_LIST ${SUPPORTED_VERSIONS} ${ARGN})
14 + if (NOT _VER IN_LIST _SUPPORTED_VERSIONS_LIST)
15 + message(FATAL_ERROR
16 + "Python version (${_VER}) is not one of the supported versions: "
17 + "${_SUPPORTED_VERSIONS_LIST}.")
18 + endif()
19 + message(STATUS "Found python matching: ${EXECUTABLE}.")
20 +endmacro()
21 +
22 +#
23 +# Run `EXPR` in python. The standard output of python is stored in `OUT` and
24 +# has trailing whitespace stripped. If an error is encountered when running
25 +# python, a fatal message `ERR_MSG` is issued.
26 +#
27 +function (run_python OUT EXPR ERR_MSG)
28 + execute_process(
29 + COMMAND
30 + "${Python_EXECUTABLE}" "-c" "${EXPR}"
31 + OUTPUT_VARIABLE PYTHON_OUT
32 + RESULT_VARIABLE PYTHON_ERROR_CODE
33 + ERROR_VARIABLE PYTHON_STDERR
34 + OUTPUT_STRIP_TRAILING_WHITESPACE)
35 +
36 + if(NOT PYTHON_ERROR_CODE EQUAL 0)
37 + message(FATAL_ERROR "${ERR_MSG}: ${PYTHON_STDERR}")
38 + endif()
39 + set(${OUT} ${PYTHON_OUT} PARENT_SCOPE)
40 +endfunction()
41 +
42 +# Run `EXPR` in python after importing `PKG`. Use the result of this to extend
43 +# `CMAKE_PREFIX_PATH` so the torch cmake configuration can be imported.
44 +macro (append_cmake_prefix_path PKG EXPR)
45 + run_python(_PREFIX_PATH
46 + "import ${PKG}; print(${EXPR})" "Failed to locate ${PKG} path")
47 + list(APPEND CMAKE_PREFIX_PATH ${_PREFIX_PATH})
48 +endmacro()
49 +
50 +#
51 +# Add a target named `hipify${NAME}` that runs the hipify preprocessor on a set
52 +# of CUDA source files. The names of the corresponding "hipified" sources are
53 +# stored in `OUT_SRCS`.
54 +#
55 +function (hipify_sources_target OUT_SRCS NAME ORIG_SRCS)
56 + #
57 + # Split into C++ and non-C++ (i.e. CUDA) sources.
58 + #
59 + set(SRCS ${ORIG_SRCS})
60 + set(CXX_SRCS ${ORIG_SRCS})
61 + list(FILTER SRCS EXCLUDE REGEX "\.(cc)|(cpp)$")
62 + list(FILTER CXX_SRCS INCLUDE REGEX "\.(cc)|(cpp)$")
63 +
64 + #
65 + # Generate ROCm/HIP source file names from CUDA file names.
66 + # Since HIP files are generated code, they will appear in the build area
67 + # `CMAKE_CURRENT_BINARY_DIR` directory rather than the original csrc dir.
68 + #
69 + set(HIP_SRCS)
70 + foreach (SRC ${SRCS})
71 + string(REGEX REPLACE "\.cu$" "\.hip" SRC ${SRC})
72 + string(REGEX REPLACE "cuda" "hip" SRC ${SRC})
73 + list(APPEND HIP_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${SRC}")
74 + endforeach()
75 +
76 + set(CSRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/csrc)
77 + add_custom_target(
78 + hipify${NAME}
79 + COMMAND ${CMAKE_SOURCE_DIR}/cmake/hipify.py -p ${CMAKE_SOURCE_DIR}/csrc -o ${CSRC_BUILD_DIR} ${SRCS}
80 + DEPENDS ${CMAKE_SOURCE_DIR}/cmake/hipify.py ${SRCS}
81 + BYPRODUCTS ${HIP_SRCS}
82 + COMMENT "Running hipify on ${NAME} extension source files.")
83 +
84 + # Swap out original extension sources with hipified sources.
85 + list(APPEND HIP_SRCS ${CXX_SRCS})
86 + set(${OUT_SRCS} ${HIP_SRCS} PARENT_SCOPE)
87 +endfunction()
88 +
89 +#
90 +# Get additional GPU compiler flags from torch.
91 +#
92 +function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
93 + if (${GPU_LANG} STREQUAL "CUDA")
94 + #
95 + # Get common NVCC flags from torch.
96 + #
97 + run_python(GPU_FLAGS
98 + "from torch.utils.cpp_extension import COMMON_NVCC_FLAGS; print(';'.join(COMMON_NVCC_FLAGS))"
99 + "Failed to determine torch nvcc compiler flags")
100 +
101 + if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102 + list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
103 + endif()
104 +
105 + elseif(${GPU_LANG} STREQUAL "HIP")
106 + #
107 + # Get common HIP/HIPCC flags from torch.
108 + #
109 + run_python(GPU_FLAGS
110 + "import torch.utils.cpp_extension as t; print(';'.join(t.COMMON_HIP_FLAGS + t.COMMON_HIPCC_FLAGS))"
111 + "Failed to determine torch nvcc compiler flags")
112 +
113 + list(APPEND GPU_FLAGS
114 + "-DUSE_ROCM"
115 + "-U__HIP_NO_HALF_CONVERSIONS__"
116 + "-U__HIP_NO_HALF_OPERATORS__"
117 + "-fno-gpu-rdc")
118 +
119 + endif()
120 + set(${OUT_GPU_FLAGS} ${GPU_FLAGS} PARENT_SCOPE)
121 +endfunction()
122 +
123 +# Macro for converting a `gencode` version number to a cmake version number.
124 +macro(string_to_ver OUT_VER IN_STR)
125 + string(REGEX REPLACE "\([0-9]+\)\([0-9]\)" "\\1.\\2" ${OUT_VER} ${IN_STR})
126 +endmacro()
127 +
128 +#
129 +# Override the GPU architectures detected by cmake/torch and filter them by
130 +# `GPU_SUPPORTED_ARCHES`. Sets the final set of architectures in
131 +# `GPU_ARCHES`.
132 +#
133 +# Note: this is defined as a macro since it updates `CMAKE_CUDA_FLAGS`.
134 +#
135 +macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
136 + set(_GPU_SUPPORTED_ARCHES_LIST ${GPU_SUPPORTED_ARCHES} ${ARGN})
137 + message(STATUS "${GPU_LANG} supported arches: ${_GPU_SUPPORTED_ARCHES_LIST}")
138 +
139 + if (${GPU_LANG} STREQUAL "HIP")
140 + #
141 + # `GPU_ARCHES` controls the `--offload-arch` flags.
142 + # `CMAKE_HIP_ARCHITECTURES` is set up by torch and can be controlled
143 + # via the `PYTORCH_ROCM_ARCH` env variable.
144 + #
145 +
146 + #
147 + # Find the intersection of the supported + detected architectures to
148 + # set the module architecture flags.
149 + #
150 + set(${GPU_ARCHES})
151 + foreach (_ARCH ${CMAKE_HIP_ARCHITECTURES})
152 + if (_ARCH IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
153 + list(APPEND ${GPU_ARCHES} ${_ARCH})
154 + endif()
155 + endforeach()
156 +
157 + if(NOT ${GPU_ARCHES})
158 + message(FATAL_ERROR
159 + "None of the detected ROCm architectures: ${CMAKE_HIP_ARCHITECTURES} is"
160 + " supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
161 + endif()
162 +
163 + elseif(${GPU_LANG} STREQUAL "CUDA")
164 + #
165 + # Setup/process CUDA arch flags.
166 + #
167 + # The torch cmake setup hardcodes the detected architecture flags in
168 + # `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
169 + # can't modified on a per-target basis, e.g. for the `punica` extension.
170 + # So, all the `-gencode` flags need to be extracted and removed from
171 + # `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
172 + # Since it's not possible to use `target_compiler_options` for adding target
173 + # specific `-gencode` arguments, the target's `CUDA_ARCHITECTURES` property
174 + # must be used instead. This requires repackaging the architecture flags
175 + # into a format that cmake expects for `CUDA_ARCHITECTURES`.
176 + #
177 + # This is a bit fragile in that it depends on torch using `-gencode` as opposed
178 + # to one of the other nvcc options to specify architectures.
179 + #
180 + # Note: torch uses the `TORCH_CUDA_ARCH_LIST` environment variable to override
181 + # detected architectures.
182 + #
183 + message(DEBUG "initial CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
184 +
185 + # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
186 + string(REGEX MATCHALL "-gencode arch=[^ ]+" _CUDA_ARCH_FLAGS
187 + ${CMAKE_CUDA_FLAGS})
188 +
189 + # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
190 + # and passed back via the `CUDA_ARCHITECTURES` property.
191 + string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
192 + ${CMAKE_CUDA_FLAGS})
193 +
194 + # If this error is triggered, it might mean that torch has changed how it sets
195 + # up nvcc architecture code generation flags.
196 + if (NOT _CUDA_ARCH_FLAGS)
197 + message(FATAL_ERROR
198 + "Could not find any architecture related code generation flags in "
199 + "CMAKE_CUDA_FLAGS. (${CMAKE_CUDA_FLAGS})")
200 + endif()
201 +
202 + message(DEBUG "final CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
203 + message(DEBUG "arch flags: ${_CUDA_ARCH_FLAGS}")
204 +
205 + # Initialize the architecture lists to empty.
206 + set(${GPU_ARCHES})
207 +
208 + # Process each `gencode` flag.
209 + foreach(_ARCH ${_CUDA_ARCH_FLAGS})
210 + # For each flag, extract the version number and whether it refers to PTX
211 + # or native code.
212 + # Note: if a regex matches then `CMAKE_MATCH_1` holds the binding
213 + # for that match.
214 +
215 + string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
216 + if (_COMPUTE)
217 + set(_COMPUTE ${CMAKE_MATCH_1})
218 + endif()
219 +
220 + string(REGEX MATCH "code=sm_\([0-9]+a?\)" _SM ${_ARCH})
221 + if (_SM)
222 + set(_SM ${CMAKE_MATCH_1})
223 + endif()
224 +
225 + string(REGEX MATCH "code=compute_\([0-9]+a?\)" _CODE ${_ARCH})
226 + if (_CODE)
227 + set(_CODE ${CMAKE_MATCH_1})
228 + endif()
229 +
230 + # Make sure the virtual architecture can be matched.
231 + if (NOT _COMPUTE)
232 + message(FATAL_ERROR
233 + "Could not determine virtual architecture from: ${_ARCH}.")
234 + endif()
235 +
236 + # One of sm_ or compute_ must exist.
237 + if ((NOT _SM) AND (NOT _CODE))
238 + message(FATAL_ERROR
239 + "Could not determine a codegen architecture from: ${_ARCH}.")
240 + endif()
241 +
242 + if (_SM)
243 + set(_VIRT "")
244 + set(_CODE_ARCH ${_SM})
245 + else()
246 + set(_VIRT "-virtual")
247 + set(_CODE_ARCH ${_CODE})
248 + endif()
249 +
250 + # Check if the current version is in the supported arch list.
251 + string_to_ver(_CODE_VER ${_CODE_ARCH})
252 + if (NOT _CODE_VER IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
253 + message(STATUS "discarding unsupported CUDA arch ${_VER}.")
254 + continue()
255 + endif()
256 +
257 + # Add it to the arch list.
258 + list(APPEND ${GPU_ARCHES} "${_CODE_ARCH}${_VIRT}")
259 + endforeach()
260 + endif()
261 + message(STATUS "${GPU_LANG} target arches: ${${GPU_ARCHES}}")
262 +endmacro()
263 +
264 +#
265 +# Define a target named `GPU_MOD_NAME` for a single extension. The
266 +# arguments are:
267 +#
268 +# DESTINATION <dest> - Module destination directory.
269 +# LANGUAGE <lang> - The GPU language for this module, e.g CUDA, HIP,
270 +# etc.
271 +# SOURCES <sources> - List of source files relative to CMakeLists.txt
272 +# directory.
273 +#
274 +# Optional arguments:
275 +#
276 +# ARCHITECTURES <arches> - A list of target GPU architectures in cmake
277 +# format.
278 +# Refer `CMAKE_CUDA_ARCHITECTURES` documentation
279 +# and `CMAKE_HIP_ARCHITECTURES` for more info.
280 +# ARCHITECTURES will use cmake's defaults if
281 +# not provided.
282 +# COMPILE_FLAGS <flags> - Extra compiler flags passed to NVCC/hip.
283 +# INCLUDE_DIRECTORIES <dirs> - Extra include directories.
284 +# LINK_LIBRARIES <libraries> - Extra link libraries.
285 +# WITH_SOABI - Generate library with python SOABI suffix name.
286 +#
287 +# Note: optimization level/debug info is set via cmake build type.
288 +#
289 +function (define_gpu_extension_target GPU_MOD_NAME)
290 + cmake_parse_arguments(PARSE_ARGV 1
291 + GPU
292 + "WITH_SOABI"
293 + "DESTINATION;LANGUAGE"
294 + "SOURCES;ARCHITECTURES;COMPILE_FLAGS;INCLUDE_DIRECTORIES;LIBRARIES")
295 +
296 + # Add hipify preprocessing step when building with HIP/ROCm.
297 + if (GPU_LANGUAGE STREQUAL "HIP")
298 + hipify_sources_target(GPU_SOURCES ${GPU_MOD_NAME} "${GPU_SOURCES}")
299 + endif()
300 +
301 + if (GPU_WITH_SOABI)
302 + set(GPU_WITH_SOABI WITH_SOABI)
303 + else()
304 + set(GPU_WITH_SOABI)
305 + endif()
306 +
307 + Python_add_library(${GPU_MOD_NAME} MODULE "${GPU_SOURCES}" ${GPU_WITH_SOABI})
308 +
309 + if (GPU_LANGUAGE STREQUAL "HIP")
310 + # Make this target dependent on the hipify preprocessor step.
311 + add_dependencies(${GPU_MOD_NAME} hipify${GPU_MOD_NAME})
312 + endif()
313 +
314 + if (GPU_ARCHITECTURES)
315 + set_target_properties(${GPU_MOD_NAME} PROPERTIES
316 + ${GPU_LANGUAGE}_ARCHITECTURES "${GPU_ARCHITECTURES}")
317 + endif()
318 +
319 + set_property(TARGET ${GPU_MOD_NAME} PROPERTY CXX_STANDARD 17)
320 +
321 + target_compile_options(${GPU_MOD_NAME} PRIVATE
322 + $<$<COMPILE_LANGUAGE:${GPU_LANGUAGE}>:${GPU_COMPILE_FLAGS}>)
323 +
324 + target_compile_definitions(${GPU_MOD_NAME} PRIVATE
325 + "-DTORCH_EXTENSION_NAME=${GPU_MOD_NAME}")
326 +
327 + target_include_directories(${GPU_MOD_NAME} PRIVATE csrc
328 + ${GPU_INCLUDE_DIRECTORIES})
329 +
330 + target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES}
331 + ${GPU_LIBRARIES})
332 +
333 + install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION})
334 +endfunction()
pyproject.toml
+1
@@ -1,6 +1,7 @@
1 [build-system]
2 # Should be mirrored in requirements-build.txt
3 requires = [
4 + "cmake>=3.21",
5 "ninja",
6 "packaging",
7 "setuptools >= 49.4.0",
requirements-build.txt
+2 -1
@@ -1,6 +1,7 @@
1 # Should be mirrored in pyproject.toml
2 +cmake>=3.21
3 ninja
4 packaging
5 setuptools>=49.4.0
6 torch==2.1.2
6 -wheel
\ No newline at end of file
7 +wheel
requirements-rocm.txt
+1
@@ -1,3 +1,4 @@
1 +cmake>=3.21
2 ninja # For faster builds.
3 typing-extensions>=4.8.0
4 starlette
requirements.txt
+1
@@ -1,3 +1,4 @@
1 +cmake>=3.21
2 ninja # For faster builds.
3 psutil
4 ray >= 2.9
setup.py
+173 -301
@@ -1,23 +1,16 @@
1 -import contextlib
1 import io
2 import os
3 import re
4 import subprocess
6 -import warnings
7 -from pathlib import Path
8 -from typing import List, Set
5 +import sys
6 +from typing import List
7
8 from packaging.version import parse, Version
11 -import setuptools
12 -import sys
9 +from setuptools import setup, find_packages, Extension
10 +from setuptools.command.build_ext import build_ext
11 +from shutil import which
12 import torch
14 -import torch.utils.cpp_extension as torch_cpp_ext
15 -from torch.utils.cpp_extension import (
16 - BuildExtension,
17 - CUDAExtension,
18 - CUDA_HOME,
19 - ROCM_HOME,
20 -)
13 +from torch.utils.cpp_extension import CUDA_HOME
14
15 ROOT_DIR = os.path.dirname(__file__)
16
@@ -25,17 +18,153 @@ ROOT_DIR = os.path.dirname(__file__)
18 assert sys.platform.startswith(
19 "linux"), "vLLM only supports Linux platform (including WSL)."
20
28 -# If you are developing the C++ backend of vLLM, consider building vLLM with
29 -# `python setup.py develop` since it will give you incremental builds.
30 -# The downside is that this method is deprecated, see
31 -# https://github.com/pypa/setuptools/issues/917
32 -
21 MAIN_CUDA_VERSION = "12.1"
22
35 -# Supported NVIDIA GPU architectures.
36 -NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
37 -ROCM_SUPPORTED_ARCHS = {"gfx908", "gfx90a", "gfx942", "gfx1100"}
38 -# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
23 +
24 +def is_sccache_available() -> bool:
25 + return which("sccache") is not None
26 +
27 +
28 +def is_ccache_available() -> bool:
29 + return which("ccache") is not None
30 +
31 +
32 +def is_ninja_available() -> bool:
33 + return which("ninja") is not None
34 +
35 +
36 +def remove_prefix(text, prefix):
37 + if text.startswith(prefix):
38 + return text[len(prefix):]
39 + return text
40 +
41 +
42 +class CMakeExtension(Extension):
43 +
44 + def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
45 + super().__init__(name, sources=[], **kwa)
46 + self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
47 +
48 +
49 +class cmake_build_ext(build_ext):
50 + # A dict of extension directories that have been configured.
51 + did_config = {}
52 +
53 + #
54 + # Determine number of compilation jobs and optionally nvcc compile threads.
55 + #
56 + def compute_num_jobs(self):
57 + try:
58 + # os.sched_getaffinity() isn't universally available, so fall back
59 + # to os.cpu_count() if we get an error here.
60 + num_jobs = len(os.sched_getaffinity(0))
61 + except AttributeError:
62 + num_jobs = os.cpu_count()
63 +
64 + nvcc_cuda_version = get_nvcc_cuda_version()
65 + if nvcc_cuda_version >= Version("11.2"):
66 + nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
67 + num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
68 + else:
69 + nvcc_threads = None
70 +
71 + return num_jobs, nvcc_threads
72 +
73 + #
74 + # Perform cmake configuration for a single extension.
75 + #
76 + def configure(self, ext: CMakeExtension) -> None:
77 + # If we've already configured using the CMakeLists.txt for
78 + # this extension, exit early.
79 + if ext.cmake_lists_dir in cmake_build_ext.did_config:
80 + return
81 +
82 + cmake_build_ext.did_config[ext.cmake_lists_dir] = True
83 +
84 + # Select the build type.
85 + # Note: optimization level + debug info are set by the build type
86 + default_cfg = "Debug" if self.debug else "RelWithDebInfo"
87 + cfg = os.getenv("CMAKE_BUILD_TYPE", default_cfg)
88 +
89 + # where .so files will be written, should be the same for all extensions
90 + # that use the same CMakeLists.txt.
91 + outdir = os.path.abspath(
92 + os.path.dirname(self.get_ext_fullpath(ext.name)))
93 +
94 + cmake_args = [
95 + '-DCMAKE_BUILD_TYPE={}'.format(cfg),
96 + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
97 + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
98 + ]
99 +
100 + verbose = bool(int(os.getenv('VERBOSE', '0')))
101 + if verbose:
102 + cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']
103 +
104 + if is_sccache_available():
105 + cmake_args += [
106 + '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
107 + '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
108 + ]
109 + elif is_ccache_available():
110 + cmake_args += [
111 + '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
112 + '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
113 + ]
114 +
115 + # Pass the python executable to cmake so it can find an exact
116 + # match.
117 + cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]
118 +
119 + if _install_punica():
120 + cmake_args += ['-DVLLM_INSTALL_PUNICA_KERNELS=ON']
121 +
122 + #
123 + # Setup parallelism and build tool
124 + #
125 + num_jobs, nvcc_threads = self.compute_num_jobs()
126 +
127 + if nvcc_threads:
128 + cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]
129 +
130 + if is_ninja_available():
131 + build_tool = ['-G', 'Ninja']
132 + cmake_args += [
133 + '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
134 + '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
135 + ]
136 + else:
137 + # Default build tool to whatever cmake picks.
138 + build_tool = []
139 +
140 + subprocess.check_call(
141 + ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
142 + cwd=self.build_temp)
143 +
144 + def build_extensions(self) -> None:
145 + # Ensure that CMake is present and working
146 + try:
147 + subprocess.check_output(['cmake', '--version'])
148 + except OSError as e:
149 + raise RuntimeError('Cannot find CMake executable') from e
150 +
151 + # Create build directory if it does not exist.
152 + if not os.path.exists(self.build_temp):
153 + os.makedirs(self.build_temp)
154 +
155 + # Build all the extensions
156 + for ext in self.extensions:
157 + self.configure(ext)
158 +
159 + ext_target_name = remove_prefix(ext.name, "vllm.")
160 + num_jobs, _ = self.compute_num_jobs()
161 +
162 + build_args = [
163 + '--build', '.', '--target', ext_target_name, '-j',
164 + str(num_jobs)
165 + ]
166 +
167 + subprocess.check_call(['cmake', *build_args], cwd=self.build_temp)
168
169
170 def _is_cuda() -> bool:
@@ -55,26 +184,8 @@ def _is_neuron() -> bool:
184 return torch_neuronx_installed
185
186
58 -# Compiler flags.
59 -CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
60 -# TODO(woosuk): Should we use -O3?
61 -NVCC_FLAGS = ["-O2", "-std=c++17"]
62 -
63 -if _is_hip():
64 - if ROCM_HOME is None:
65 - raise RuntimeError("Cannot find ROCM_HOME. "
66 - "ROCm must be available to build the package.")
67 - NVCC_FLAGS += ["-DUSE_ROCM"]
68 - NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
69 - NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
70 -
71 -if _is_cuda() and CUDA_HOME is None:
72 - raise RuntimeError(
73 - "Cannot find CUDA_HOME. CUDA must be available to build the package.")
74 -
75 -ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
76 -CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
77 -NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
187 +def _install_punica() -> bool:
188 + return bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
189
190
191 def get_hipcc_rocm_version():
@@ -99,11 +210,6 @@ def get_hipcc_rocm_version():
210 return None
211
212
102 -def glob(pattern: str):
103 - root = Path(__name__).parent
104 - return [str(p) for p in root.glob(pattern)]
105 -
106 -
213 def get_neuronxcc_version():
214 import sysconfig
215 site_dir = sysconfig.get_paths()["purelib"]
@@ -123,12 +229,12 @@ def get_neuronxcc_version():
229 raise RuntimeError("Could not find HIP version in the output")
230
231
126 -def get_nvcc_cuda_version(cuda_dir: str) -> Version:
232 +def get_nvcc_cuda_version() -> Version:
233 """Get the CUDA version from nvcc.
234
235 Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
236 """
131 - nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
237 + nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
238 universal_newlines=True)
239 output = nvcc_output.split()
240 release_idx = output.index("release") + 1
@@ -136,250 +242,6 @@ def get_nvcc_cuda_version(cuda_dir: str) -> Version:
242 return nvcc_cuda_version
243
244
139 -def get_pytorch_rocm_arch() -> Set[str]:
140 - """Get the cross section of Pytorch,and vllm supported gfx arches
141 -
142 - ROCM can get the supported gfx architectures in one of two ways
143 - Either through the PYTORCH_ROCM_ARCH env var, or output from
144 - rocm_agent_enumerator.
145 -
146 - In either case we can generate a list of supported arch's and
147 - cross reference with VLLM's own ROCM_SUPPORTED_ARCHs.
148 - """
149 - env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
150 -
151 - # If we don't have PYTORCH_ROCM_ARCH specified pull the list from
152 - # rocm_agent_enumerator
153 - if env_arch_list is None:
154 - command = "rocm_agent_enumerator"
155 - env_arch_list = (subprocess.check_output(
156 - [command]).decode('utf-8').strip().replace("\n", ";"))
157 - arch_source_str = "rocm_agent_enumerator"
158 - else:
159 - arch_source_str = "PYTORCH_ROCM_ARCH env variable"
160 -
161 - # List are separated by ; or space.
162 - pytorch_rocm_arch = set(env_arch_list.replace(" ", ";").split(";"))
163 -
164 - # Filter out the invalid architectures and print a warning.
165 - arch_list = pytorch_rocm_arch.intersection(ROCM_SUPPORTED_ARCHS)
166 -
167 - # If none of the specified architectures are valid, raise an error.
168 - if not arch_list:
169 - raise RuntimeError(
170 - f"None of the ROCM architectures in {arch_source_str} "
171 - f"({env_arch_list}) is supported. "
172 - f"Supported ROCM architectures are: {ROCM_SUPPORTED_ARCHS}.")
173 - invalid_arch_list = pytorch_rocm_arch - ROCM_SUPPORTED_ARCHS
174 - if invalid_arch_list:
175 - warnings.warn(
176 - f"Unsupported ROCM architectures ({invalid_arch_list}) are "
177 - f"excluded from the {arch_source_str} output "
178 - f"({env_arch_list}). Supported ROCM architectures are: "
179 - f"{ROCM_SUPPORTED_ARCHS}.",
180 - stacklevel=2)
181 - return arch_list
182 -
183 -
184 -def get_torch_arch_list() -> Set[str]:
185 - # TORCH_CUDA_ARCH_LIST can have one or more architectures,
186 - # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
187 - # compiler to additionally include PTX code that can be runtime-compiled
188 - # and executed on the 8.6 or newer architectures. While the PTX code will
189 - # not give the best performance on the newer architectures, it provides
190 - # forward compatibility.
191 - env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
192 - if env_arch_list is None:
193 - return set()
194 -
195 - # List are separated by ; or space.
196 - torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
197 - if not torch_arch_list:
198 - return set()
199 -
200 - # Filter out the invalid architectures and print a warning.
201 - valid_archs = NVIDIA_SUPPORTED_ARCHS.union(
202 - {s + "+PTX"
203 - for s in NVIDIA_SUPPORTED_ARCHS})
204 - arch_list = torch_arch_list.intersection(valid_archs)
205 - # If none of the specified architectures are valid, raise an error.
206 - if not arch_list:
207 - raise RuntimeError(
208 - "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
209 - f"variable ({env_arch_list}) is supported. "
210 - f"Supported CUDA architectures are: {valid_archs}.")
211 - invalid_arch_list = torch_arch_list - valid_archs
212 - if invalid_arch_list:
213 - warnings.warn(
214 - f"Unsupported CUDA architectures ({invalid_arch_list}) are "
215 - "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
216 - f"({env_arch_list}). Supported CUDA architectures are: "
217 - f"{valid_archs}.",
218 - stacklevel=2)
219 - return arch_list
220 -
221 -
222 -if _is_hip():
223 - rocm_arches = get_pytorch_rocm_arch()
224 - NVCC_FLAGS += ["--offload-arch=" + arch for arch in rocm_arches]
225 -else:
226 - # First, check the TORCH_CUDA_ARCH_LIST environment variable.
227 - compute_capabilities = get_torch_arch_list()
228 -
229 -if _is_cuda() and not compute_capabilities:
230 - # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
231 - # GPUs on the current machine.
232 - device_count = torch.cuda.device_count()
233 - for i in range(device_count):
234 - major, minor = torch.cuda.get_device_capability(i)
235 - if major < 7:
236 - raise RuntimeError(
237 - "GPUs with compute capability below 7.0 are not supported.")
238 - compute_capabilities.add(f"{major}.{minor}")
239 -
240 -ext_modules = []
241 -
242 -if _is_cuda():
243 - nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
244 - if not compute_capabilities:
245 - # If no GPU is specified nor available, add all supported architectures
246 - # based on the NVCC CUDA version.
247 - compute_capabilities = NVIDIA_SUPPORTED_ARCHS.copy()
248 - if nvcc_cuda_version < Version("11.1"):
249 - compute_capabilities.remove("8.6")
250 - if nvcc_cuda_version < Version("11.8"):
251 - compute_capabilities.remove("8.9")
252 - compute_capabilities.remove("9.0")
253 - # Validate the NVCC CUDA version.
254 - if nvcc_cuda_version < Version("11.0"):
255 - raise RuntimeError(
256 - "CUDA 11.0 or higher is required to build the package.")
257 - if (nvcc_cuda_version < Version("11.1")
258 - and any(cc.startswith("8.6") for cc in compute_capabilities)):
259 - raise RuntimeError(
260 - "CUDA 11.1 or higher is required for compute capability 8.6.")
261 - if nvcc_cuda_version < Version("11.8"):
262 - if any(cc.startswith("8.9") for cc in compute_capabilities):
263 - # CUDA 11.8 is required to generate the code targeting compute
264 - # capability 8.9. However, GPUs with compute capability 8.9 can
265 - # also run the code generated by the previous versions of CUDA 11
266 - # and targeting compute capability 8.0. Therefore, if CUDA 11.8
267 - # is not available, we target compute capability 8.0 instead of 8.9.
268 - warnings.warn(
269 - "CUDA 11.8 or higher is required for compute capability 8.9. "
270 - "Targeting compute capability 8.0 instead.",
271 - stacklevel=2)
272 - compute_capabilities = set(cc for cc in compute_capabilities
273 - if not cc.startswith("8.9"))
274 - compute_capabilities.add("8.0+PTX")
275 - if any(cc.startswith("9.0") for cc in compute_capabilities):
276 - raise RuntimeError(
277 - "CUDA 11.8 or higher is required for compute capability 9.0.")
278 -
279 - NVCC_FLAGS_PUNICA = NVCC_FLAGS.copy()
280 -
281 - # Add target compute capabilities to NVCC flags.
282 - for capability in compute_capabilities:
283 - num = capability[0] + capability[2]
284 - NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
285 - if capability.endswith("+PTX"):
286 - NVCC_FLAGS += [
287 - "-gencode", f"arch=compute_{num},code=compute_{num}"
288 - ]
289 - if int(capability[0]) >= 8:
290 - NVCC_FLAGS_PUNICA += [
291 - "-gencode", f"arch=compute_{num},code=sm_{num}"
292 - ]
293 - if capability.endswith("+PTX"):
294 - NVCC_FLAGS_PUNICA += [
295 - "-gencode", f"arch=compute_{num},code=compute_{num}"
296 - ]
297 -
298 - # Use NVCC threads to parallelize the build.
299 - if nvcc_cuda_version >= Version("11.2"):
300 - nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
301 - num_threads = min(os.cpu_count(), nvcc_threads)
302 - NVCC_FLAGS += ["--threads", str(num_threads)]
303 -
304 - if nvcc_cuda_version >= Version("11.8"):
305 - NVCC_FLAGS += ["-DENABLE_FP8_E5M2"]
306 -
307 - # changes for punica kernels
308 - NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
309 - REMOVE_NVCC_FLAGS = [
310 - '-D__CUDA_NO_HALF_OPERATORS__',
311 - '-D__CUDA_NO_HALF_CONVERSIONS__',
312 - '-D__CUDA_NO_BFLOAT16_CONVERSIONS__',
313 - '-D__CUDA_NO_HALF2_OPERATORS__',
314 - ]
315 - for flag in REMOVE_NVCC_FLAGS:
316 - with contextlib.suppress(ValueError):
317 - torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
318 -
319 - install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
320 - device_count = torch.cuda.device_count()
321 - for i in range(device_count):
322 - major, minor = torch.cuda.get_device_capability(i)
323 - if major < 8:
324 - install_punica = False
325 - break
326 - if install_punica:
327 - ext_modules.append(
328 - CUDAExtension(
329 - name="vllm._punica_C",
330 - sources=["csrc/punica/punica_ops.cc"] +
331 - glob("csrc/punica/bgmv/*.cu"),
332 - extra_compile_args={
333 - "cxx": CXX_FLAGS,
334 - "nvcc": NVCC_FLAGS_PUNICA,
335 - },
336 - ))
337 -elif _is_neuron():
338 - neuronxcc_version = get_neuronxcc_version()
339 -
340 -vllm_extension_sources = [
341 - "csrc/cache_kernels.cu",
342 - "csrc/attention/attention_kernels.cu",
343 - "csrc/pos_encoding_kernels.cu",
344 - "csrc/activation_kernels.cu",
345 - "csrc/layernorm_kernels.cu",
346 - "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
347 - "csrc/quantization/gptq/q_gemm.cu",
348 - "csrc/cuda_utils_kernels.cu",
349 - "csrc/moe_align_block_size_kernels.cu",
350 - "csrc/pybind.cpp",
351 -]
352 -
353 -if _is_cuda():
354 - vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
355 - vllm_extension_sources.append(
356 - "csrc/quantization/marlin/marlin_cuda_kernel.cu")
357 - vllm_extension_sources.append("csrc/custom_all_reduce.cu")
358 -
359 - # Add MoE kernels.
360 - ext_modules.append(
361 - CUDAExtension(
362 - name="vllm._moe_C",
363 - sources=glob("csrc/moe/*.cu") + glob("csrc/moe/*.cpp"),
364 - extra_compile_args={
365 - "cxx": CXX_FLAGS,
366 - "nvcc": NVCC_FLAGS,
367 - },
368 - ))
369 -
370 -if not _is_neuron():
371 - vllm_extension = CUDAExtension(
372 - name="vllm._C",
373 - sources=vllm_extension_sources,
374 - extra_compile_args={
375 - "cxx": CXX_FLAGS,
376 - "nvcc": NVCC_FLAGS,
377 - },
378 - libraries=["cuda"] if _is_cuda() else [],
379 - )
380 - ext_modules.append(vllm_extension)
381 -
382 -
245 def get_path(*filepath) -> str:
246 return os.path.join(ROOT_DIR, *filepath)
247
@@ -401,7 +263,7 @@ def get_vllm_version() -> str:
263 version = find_version(get_path("vllm", "__init__.py"))
264
265 if _is_cuda():
404 - cuda_version = str(nvcc_cuda_version)
266 + cuda_version = str(get_nvcc_cuda_version())
267 if cuda_version != MAIN_CUDA_VERSION:
268 cuda_version_str = cuda_version.replace(".", "")[:3]
269 version += f"+cu{cuda_version_str}"
@@ -413,7 +275,7 @@ def get_vllm_version() -> str:
275 version += f"+rocm{rocm_version_str}"
276 elif _is_neuron():
277 # Get the Neuron version
416 - neuron_version = str(neuronxcc_version)
278 + neuron_version = str(get_neuronxcc_version())
279 if neuron_version != MAIN_CUDA_VERSION:
280 neuron_version_str = neuron_version.replace(".", "")[:3]
281 version += f"+neuron{neuron_version_str}"
@@ -437,7 +299,7 @@ def get_requirements() -> List[str]:
299 if _is_cuda():
300 with open(get_path("requirements.txt")) as f:
301 requirements = f.read().strip().split("\n")
440 - if nvcc_cuda_version <= Version("11.8"):
302 + if get_nvcc_cuda_version() <= Version("11.8"):
303 # replace cupy-cuda12x with cupy-cuda11x for cuda 11.x
304 for i in range(len(requirements)):
305 if requirements[i].startswith("cupy-cuda12x"):
@@ -456,14 +318,24 @@ def get_requirements() -> List[str]:
318 return requirements
319
320
321 +ext_modules = []
322 +
323 +if _is_cuda():
324 + ext_modules.append(CMakeExtension(name="vllm._moe_C"))
325 +
326 + if _install_punica():
327 + ext_modules.append(CMakeExtension(name="vllm._punica_C"))
328 +
329 +if not _is_neuron():
330 + ext_modules.append(CMakeExtension(name="vllm._C"))
331 +
332 package_data = {
333 "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
334 }
335 if os.environ.get("VLLM_USE_PRECOMPILED"):
463 - ext_modules = []
336 package_data["vllm"].append("*.so")
337
466 -setuptools.setup(
338 +setup(
339 name="vllm",
340 version=get_vllm_version(),
341 author="vLLM Team",
@@ -485,11 +357,11 @@ setuptools.setup(
357 "License :: OSI Approved :: Apache Software License",
358 "Topic :: Scientific/Engineering :: Artificial Intelligence",
359 ],
488 - packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
489 - "examples", "tests")),
360 + packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
361 + "tests")),
362 python_requires=">=3.8",
363 install_requires=get_requirements(),
364 ext_modules=ext_modules,
493 - cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
365 + cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
366 package_data=package_data,
367 )