@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55d63b1211
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] Don't build machete on cuda <12.0 (#7757)

Lucas Wilkinson committed Aug 22, 2024 at 08:28 UTC 55d63b1211af9bd3a71d88f82f3bd5804d83b2cd
2 files changed +47 -27
CMakeLists.txt
+35 -27
@@ -10,6 +10,9 @@ message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
10
11 include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
12
13 +# Suppress potential warnings about unused manually-specified variables
14 +set(ignoreMe "${VLLM_PYTHON_PATH}")
15 +
16 #
17 # Supported python versions. These versions will be searched in order, the
18 # first match will be selected. These should be kept in sync with setup.py.
@@ -228,35 +231,38 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
231 endif()
232
233 #
231 - # For the Machete kernels we automatically generate sources for various
232 - # preselected input type pairs and schedules.
233 - # Generate sources:
234 - execute_process(
235 - COMMAND ${CMAKE_COMMAND} -E env
236 - PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
237 - ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
238 - RESULT_VARIABLE machete_generation_result
239 - OUTPUT_VARIABLE machete_generation_output
240 - OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
241 - ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
242 - )
234 + # Machete kernels
235
244 - if (NOT machete_generation_result EQUAL 0)
245 - message(FATAL_ERROR "Machete generation failed."
246 - " Result: \"${machete_generation_result}\""
247 - "\nCheck the log for details: "
248 - "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
249 - else()
250 - message(STATUS "Machete generation completed successfully.")
251 - endif()
236 + # The machete kernels only work on hopper and require CUDA 12.0 or later.
237 + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
238 + #
239 + # For the Machete kernels we automatically generate sources for various
240 + # preselected input type pairs and schedules.
241 + # Generate sources:
242 + execute_process(
243 + COMMAND ${CMAKE_COMMAND} -E env
244 + PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
245 + ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
246 + RESULT_VARIABLE machete_generation_result
247 + OUTPUT_VARIABLE machete_generation_output
248 + OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
249 + ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
250 + )
251 +
252 + if (NOT machete_generation_result EQUAL 0)
253 + message(FATAL_ERROR "Machete generation failed."
254 + " Result: \"${machete_generation_result}\""
255 + "\nCheck the log for details: "
256 + "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
257 + else()
258 + message(STATUS "Machete generation completed successfully.")
259 + endif()
260
253 - # Add machete generated sources
254 - file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
255 - list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
256 - message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
261 + # Add machete generated sources
262 + file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
263 + list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
264 + message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
265
258 - # See comment above for scaled_mm_c3x (same if condition)
259 - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
266 set_source_files_properties(
267 ${MACHETE_GEN_SOURCES}
268 PROPERTIES
@@ -264,7 +270,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
270 "-gencode arch=compute_90a,code=sm_90a")
271 endif()
272
267 - # Add pytorch binding
273 + # Add pytorch binding for machete (add on even CUDA < 12.0 so that we can
274 + # raise an error if the user that this was built with an incompatible
275 + # CUDA version)
276 list(APPEND VLLM_EXT_SRC
277 csrc/quantization/machete/machete_pytorch.cu)
278 endif()
csrc/quantization/machete/machete_pytorch.cu
+12
@@ -37,9 +37,13 @@ static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
37 //
38
39 std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
40 +#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
41 return scalar_type_dispatch(*btype, [&](auto BType) {
42 return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
43 });
44 +#else
45 + TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
46 +#endif
47 }
48
49 torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
@@ -50,6 +54,7 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
54 c10::optional<torch::Tensor> const& C,
55 c10::optional<double> alpha, c10::optional<double> beta,
56 c10::optional<std::string> schedule) {
57 +#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
58 auto args = PyTorchArguments{.A = A,
59 .B = B,
60 .scales = scales,
@@ -67,13 +72,20 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
72 return GemmDispatcher<ComputeType, decltype(BType)>::dispatch(args);
73 });
74 });
75 +#else
76 + TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
77 +#endif
78 }
79
80 torch::Tensor prepack_B(torch::Tensor const& B,
81 ScalarTypeTorchPtr const& btype) {
82 +#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
83 return scalar_type_dispatch(*btype, [&](auto BType) {
84 return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
85 });
86 +#else
87 + TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
88 +#endif
89 }
90
91 }; // namespace machete