@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5467ac3196
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel][Misc] Use TORCH_LIBRARY instead of PYBIND11_MODULE for custom ops (#5047)

bnellnm committed Jun 9, 2024 at 16:23 UTC 5467ac319636245ded483b31967ac43e543c5fa3
55 files changed +833 -451
CMakeLists.txt
+6 -16
@@ -66,19 +66,6 @@ endif()
66 #
67 find_package(Torch REQUIRED)
68
69 -#
70 -# Normally `torch.utils.cpp_extension.CUDAExtension` would add
71 -# `libtorch_python.so` for linking against an extension. Torch's cmake
72 -# configuration does not include this library (presumably since the cmake
73 -# config is used for standalone C++ binaries that link against torch).
74 -# The `libtorch_python.so` library defines some of the glue code between
75 -# torch/python via pybind and is required by VLLM extensions for this
76 -# reason. So, add it by manually with `find_library` using torch's
77 -# installed library path.
78 -#
79 -find_library(torch_python_LIBRARY torch_python PATHS
80 - "${TORCH_INSTALL_PREFIX}/lib")
81 -
69 #
70 # Forward the non-CUDA device extensions to external CMake scripts.
71 #
@@ -171,7 +158,7 @@ set(VLLM_EXT_SRC
158 "csrc/quantization/fp8/common.cu"
159 "csrc/cuda_utils_kernels.cu"
160 "csrc/moe_align_block_size_kernels.cu"
174 - "csrc/pybind.cpp")
161 + "csrc/torch_bindings.cpp")
162
163 if(VLLM_GPU_LANG STREQUAL "CUDA")
164 include(FetchContent)
@@ -218,6 +205,7 @@ define_gpu_extension_target(
205 COMPILE_FLAGS ${VLLM_GPU_FLAGS}
206 ARCHITECTURES ${VLLM_GPU_ARCHES}
207 INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
208 + USE_SABI 3
209 WITH_SOABI)
210
211 #
@@ -225,7 +213,7 @@ define_gpu_extension_target(
213 #
214
215 set(VLLM_MOE_EXT_SRC
228 - "csrc/moe/moe_ops.cpp"
216 + "csrc/moe/torch_bindings.cpp"
217 "csrc/moe/topk_softmax_kernels.cu")
218
219 define_gpu_extension_target(
@@ -235,6 +223,7 @@ define_gpu_extension_target(
223 SOURCES ${VLLM_MOE_EXT_SRC}
224 COMPILE_FLAGS ${VLLM_GPU_FLAGS}
225 ARCHITECTURES ${VLLM_GPU_ARCHES}
226 + USE_SABI 3
227 WITH_SOABI)
228
229 #
@@ -249,7 +238,7 @@ set(VLLM_PUNICA_EXT_SRC
238 "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
239 "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
240 "csrc/punica/punica_ops.cu"
252 - "csrc/punica/punica_pybind.cpp")
241 + "csrc/punica/torch_bindings.cpp")
242
243 #
244 # Copy GPU compilation flags+update for punica
@@ -286,6 +275,7 @@ if (VLLM_PUNICA_GPU_ARCHES)
275 SOURCES ${VLLM_PUNICA_EXT_SRC}
276 COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
277 ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
278 + USE_SABI 3
279 WITH_SOABI)
280 else()
281 message(WARNING "Unable to create _punica_C target because none of the "
Dockerfile.rocm
+3 -3
@@ -106,9 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
106 pip install -U -r requirements-rocm.txt \
107 && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
108 && python3 setup.py install \
109 - && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110 - && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111 - && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.cpython-39-x86_64-linux-gnu.so vllm/ \
109 + && cp build/lib.linux-x86_64-cpython-39/vllm/_C.abi3.so vllm/ \
110 + && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.abi3.so vllm/ \
111 + && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.abi3.so vllm/ \
112 && cd ..
113
114
cmake/cpu_extension.cmake
+6 -6
@@ -12,7 +12,7 @@ include_directories("${CMAKE_SOURCE_DIR}/csrc")
12 #
13 # Check the compile flags
14 #
15 -list(APPEND CXX_COMPILE_FLAGS
15 +list(APPEND CXX_COMPILE_FLAGS
16 "-fopenmp"
17 "-DVLLM_CPU_EXTENSION")
18
@@ -44,8 +44,8 @@ if (AVX512_FOUND)
44
45 find_isa(${CPUINFO} "avx512_bf16" AVX512BF16_FOUND)
46 if (AVX512BF16_FOUND OR ENABLE_AVX512BF16)
47 - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
48 - CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)
47 + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
48 + CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)
49 list(APPEND CXX_COMPILE_FLAGS "-mavx512bf16")
50 else()
51 message(WARNING "Disable AVX512-BF16 ISA support, requires gcc/g++ >= 12.3")
@@ -73,7 +73,7 @@ set(VLLM_EXT_SRC
73 "csrc/cpu/cache.cpp"
74 "csrc/cpu/layernorm.cpp"
75 "csrc/cpu/pos_encoding.cpp"
76 - "csrc/cpu/pybind.cpp")
76 + "csrc/cpu/torch_bindings.cpp")
77
78 define_gpu_extension_target(
79 _C
@@ -81,10 +81,10 @@ define_gpu_extension_target(
81 LANGUAGE CXX
82 SOURCES ${VLLM_EXT_SRC}
83 COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
84 - WITH_SOABI
84 + USE_SABI 3
85 + WITH_SOABI
86 )
87
88 add_custom_target(default)
89 message(STATUS "Enabling C extension.")
90 add_dependencies(default _C)
90 -
cmake/utils.cmake
+8 -3
@@ -5,7 +5,7 @@
5 macro (find_python_from_executable EXECUTABLE SUPPORTED_VERSIONS)
6 file(REAL_PATH ${EXECUTABLE} EXECUTABLE)
7 set(Python_EXECUTABLE ${EXECUTABLE})
8 - find_package(Python COMPONENTS Interpreter Development.Module)
8 + find_package(Python COMPONENTS Interpreter Development.Module Development.SABIModule)
9 if (NOT Python_FOUND)
10 message(FATAL_ERROR "Unable to find python matching: ${EXECUTABLE}.")
11 endif()
@@ -294,6 +294,7 @@ endmacro()
294 # INCLUDE_DIRECTORIES <dirs> - Extra include directories.
295 # LIBRARIES <libraries> - Extra link libraries.
296 # WITH_SOABI - Generate library with python SOABI suffix name.
297 +# USE_SABI <version> - Use python stable api <version>
298 #
299 # Note: optimization level/debug info is set via cmake build type.
300 #
@@ -301,7 +302,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
302 cmake_parse_arguments(PARSE_ARGV 1
303 GPU
304 "WITH_SOABI"
304 - "DESTINATION;LANGUAGE"
305 + "DESTINATION;LANGUAGE;USE_SABI"
306 "SOURCES;ARCHITECTURES;COMPILE_FLAGS;INCLUDE_DIRECTORIES;LIBRARIES")
307
308 # Add hipify preprocessing step when building with HIP/ROCm.
@@ -315,7 +316,11 @@ function (define_gpu_extension_target GPU_MOD_NAME)
316 set(GPU_WITH_SOABI)
317 endif()
318
318 - Python_add_library(${GPU_MOD_NAME} MODULE "${GPU_SOURCES}" ${GPU_WITH_SOABI})
319 + if (GPU_USE_SABI)
320 + Python_add_library(${GPU_MOD_NAME} MODULE USE_SABI ${GPU_USE_SABI} ${GPU_WITH_SOABI} "${GPU_SOURCES}")
321 + else()
322 + Python_add_library(${GPU_MOD_NAME} MODULE ${GPU_WITH_SOABI} "${GPU_SOURCES}")
323 + endif()
324
325 if (GPU_LANGUAGE STREQUAL "HIP")
326 # Make this target dependent on the hipify preprocessor step.
csrc/activation_kernels.cu
+1 -1
@@ -1,5 +1,5 @@
1 #include <ATen/cuda/CUDAContext.h>
2 -#include <torch/extension.h>
2 +#include <torch/all.h>
3 #include <c10/cuda/CUDAGuard.h>
4
5 #include <cmath>
csrc/attention/attention_kernels.cu
+18 -16
@@ -17,7 +17,7 @@
17 * limitations under the License.
18 */
19
20 -#include <torch/extension.h>
20 +#include <torch/all.h>
21 #include <ATen/cuda/CUDAContext.h>
22 #include <c10/cuda/CUDAGuard.h>
23 #include <algorithm>
@@ -808,16 +808,17 @@ void paged_attention_v1(
808 torch::Tensor&
809 key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
810 torch::Tensor&
811 - value_cache, // [num_blocks, num_heads, head_size, block_size]
812 - int num_kv_heads, // [num_heads]
813 - float scale,
811 + value_cache, // [num_blocks, num_heads, head_size, block_size]
812 + int64_t num_kv_heads, // [num_heads]
813 + double scale,
814 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
815 torch::Tensor& seq_lens, // [num_seqs]
816 - int block_size, int max_seq_len,
816 + int64_t block_size, int64_t max_seq_len,
817 const c10::optional<torch::Tensor>& alibi_slopes,
818 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
819 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
820 - const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
818 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
819 + const int64_t blocksparse_local_blocks,
820 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
821 + const int64_t blocksparse_head_sliding_step) {
822 const bool is_block_sparse = (blocksparse_vert_stride > 1);
823
824 DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
@@ -972,16 +973,17 @@ void paged_attention_v2(
973 torch::Tensor&
974 key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
975 torch::Tensor&
975 - value_cache, // [num_blocks, num_heads, head_size, block_size]
976 - int num_kv_heads, // [num_heads]
977 - float scale,
976 + value_cache, // [num_blocks, num_heads, head_size, block_size]
977 + int64_t num_kv_heads, // [num_heads]
978 + double scale,
979 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
980 torch::Tensor& seq_lens, // [num_seqs]
980 - int block_size, int max_seq_len,
981 + int64_t block_size, int64_t max_seq_len,
982 const c10::optional<torch::Tensor>& alibi_slopes,
982 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
983 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
984 - const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
983 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
984 + const int64_t blocksparse_local_blocks,
985 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
986 + const int64_t blocksparse_head_sliding_step) {
987 const bool is_block_sparse = (blocksparse_vert_stride > 1);
988 DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
989 CALL_V2_LAUNCHER_BLOCK_SIZE)
@@ -990,4 +992,4 @@ void paged_attention_v2(
992 #undef WARP_SIZE
993 #undef MAX
994 #undef MIN
993 -#undef DIVIDE_ROUND_UP
\ No newline at end of file
995 +#undef DIVIDE_ROUND_UP
csrc/cache.h
+9 -5
@@ -1,6 +1,6 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +#include <torch/all.h>
4
5 #include <map>
6 #include <vector>
@@ -8,14 +8,18 @@
8 void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
9 const torch::Tensor& block_mapping);
10
11 -void copy_blocks(std::vector<torch::Tensor>& key_caches,
12 - std::vector<torch::Tensor>& value_caches,
11 +// Note: the key_caches and value_caches vectors are constant but
12 +// not the Tensors they contain. The vectors need to be const refs
13 +// in order to satisfy pytorch's C++ operator registration code.
14 +void copy_blocks(std::vector<torch::Tensor> const& key_caches,
15 + std::vector<torch::Tensor> const& value_caches,
16 const torch::Tensor& block_mapping);
17
18 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19 torch::Tensor& key_cache, torch::Tensor& value_cache,
20 torch::Tensor& slot_mapping,
18 - const std::string& kv_cache_dtype, const float kv_scale);
21 + const std::string& kv_cache_dtype,
22 + const double kv_scale);
23
24 void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25 torch::Tensor& key_cache,
@@ -25,4 +29,4 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
29
30 // Just for unittest
31 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
28 - const float scale, const std::string& kv_cache_dtype);
32 + const double scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+8 -5
@@ -1,4 +1,4 @@
1 -#include <torch/extension.h>
1 +#include <torch/all.h>
2 #include <ATen/cuda/CUDAContext.h>
3 #include <c10/cuda/CUDAGuard.h>
4
@@ -95,8 +95,11 @@ __global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
95
96 } // namespace vllm
97
98 -void copy_blocks(std::vector<torch::Tensor>& key_caches,
99 - std::vector<torch::Tensor>& value_caches,
98 +// Note: the key_caches and value_caches vectors are constant but
99 +// not the Tensors they contain. The vectors need to be const refs
100 +// in order to satisfy pytorch's C++ operator registration code.
101 +void copy_blocks(std::vector<torch::Tensor> const& key_caches,
102 + std::vector<torch::Tensor> const& value_caches,
103 const torch::Tensor& block_mapping) {
104 int num_layers = key_caches.size();
105 TORCH_CHECK(num_layers == value_caches.size());
@@ -255,7 +258,7 @@ void reshape_and_cache(
258 torch::Tensor&
259 value_cache, // [num_blocks, num_heads, head_size, block_size]
260 torch::Tensor& slot_mapping, // [num_tokens]
258 - const std::string& kv_cache_dtype, const float kv_scale) {
261 + const std::string& kv_cache_dtype, const double kv_scale) {
262 int num_tokens = key.size(0);
263 int num_heads = key.size(1);
264 int head_size = key.size(2);
@@ -334,7 +337,7 @@ __global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
337
338 // Only for testing.
339 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
337 - const float kv_scale, const std::string& kv_cache_dtype) {
340 + const double kv_scale, const std::string& kv_cache_dtype) {
341 torch::Device src_device = src_cache.device();
342 torch::Device dst_device = dst_cache.device();
343 TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
csrc/cpu/attention.cpp
+14 -12
@@ -420,12 +420,13 @@ void paged_attention_v1_impl_launcher(
420
421 void paged_attention_v1(
422 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
423 - torch::Tensor& value_cache, int num_kv_heads, float scale,
424 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
425 - int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
427 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
428 - const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
423 + torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
424 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
425 + int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
427 + const int64_t blocksparse_local_blocks,
428 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
429 + const int64_t blocksparse_head_sliding_step) {
430 TORCH_CHECK(kv_scale == 1.0f);
431 TORCH_CHECK(blocksparse_vert_stride <= 1,
432 "CPU backend does not support blocksparse attention yet.");
@@ -738,12 +739,13 @@ void paged_attention_v2_impl_launcher(
739 void paged_attention_v2(
740 torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
741 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
741 - torch::Tensor& value_cache, int num_kv_heads, float scale,
742 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
743 - int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
744 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
745 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
746 - const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
742 + torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
743 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
744 + int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
745 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
746 + const int64_t blocksparse_local_blocks,
747 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
748 + const int64_t blocksparse_head_sliding_step) {
749 TORCH_CHECK(kv_scale == 1.0f);
750 TORCH_CHECK(blocksparse_vert_stride <= 1,
751 "CPU backend does not support blocksparse attention yet.");
csrc/cpu/cache.cpp
+8 -5
@@ -5,8 +5,8 @@
5
6 namespace {
7 template <typename scalar_t>
8 -void copy_blocks_cpu_impl(std::vector<torch::Tensor>& key_caches,
9 - std::vector<torch::Tensor>& value_caches,
8 +void copy_blocks_cpu_impl(std::vector<torch::Tensor> const& key_caches,
9 + std::vector<torch::Tensor> const& value_caches,
10 const torch::Tensor& mapping_pairs,
11 const int element_num_per_block,
12 const int layer_num) {
@@ -82,8 +82,11 @@ void reshape_and_cache_cpu_impl(
82 }
83 }; // namespace
84
85 -void copy_blocks(std::vector<torch::Tensor>& key_caches,
86 - std::vector<torch::Tensor>& value_caches,
85 +// Note: the key_caches and value_caches vectors are constant but
86 +// not the Tensors they contain. The vectors need to be const refs
87 +// in order to satisfy pytorch's C++ operator registration code.
88 +void copy_blocks(std::vector<torch::Tensor> const& key_caches,
89 + std::vector<torch::Tensor> const& value_caches,
90 const torch::Tensor& block_mapping) {
91 unsigned num_layers = key_caches.size();
92 TORCH_CHECK(num_layers == value_caches.size());
@@ -104,7 +107,7 @@ void copy_blocks(std::vector<torch::Tensor>& key_caches,
107 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108 torch::Tensor& key_cache, torch::Tensor& value_cache,
109 torch::Tensor& slot_mapping,
107 - const std::string& kv_cache_dtype, float kv_scale) {
110 + const std::string& kv_cache_dtype, double kv_scale) {
111 TORCH_CHECK(kv_scale == 1.0f);
112
113 int num_tokens = key.size(0);
csrc/cpu/cpu_types.hpp
+1 -1
@@ -3,7 +3,7 @@
3 #define CPU_TYPES_HPP
4
5 #include <immintrin.h>
6 -#include <torch/extension.h>
6 +#include <torch/all.h>
7
8 namespace vec_op {
9
csrc/cpu/layernorm.cpp
+2 -2
@@ -88,7 +88,7 @@ void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
88 } // namespace
89
90 void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
91 - float epsilon) {
91 + double epsilon) {
92 int hidden_size = input.size(-1);
93 int num_tokens = input.numel() / hidden_size;
94
@@ -102,7 +102,7 @@ void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
102 }
103
104 void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
105 - torch::Tensor& weight, float epsilon) {
105 + torch::Tensor& weight, double epsilon) {
106 int hidden_size = input.size(-1);
107 int num_tokens = input.numel() / hidden_size;
108
csrc/cpu/pos_encoding.cpp
+1 -1
@@ -168,7 +168,7 @@ void rotary_embedding_gptj_impl(
168 }; // namespace
169
170 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
171 - torch::Tensor& key, int head_size,
171 + torch::Tensor& key, int64_t head_size,
172 torch::Tensor& cos_sin_cache, bool is_neox) {
173 int num_tokens = query.numel() / query.size(-1);
174 int rot_dim = cos_sin_cache.size(1);
csrc/cpu/pybind.cpp deleted
-43
@@ -1,43 +0,0 @@
1 -#include "cache.h"
2 -#include "ops.h"
3 -#include <torch/extension.h>
4 -
5 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6 - // vLLM custom ops
7 - pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
8 -
9 - // Attention ops
10 - ops.def("paged_attention_v1", &paged_attention_v1,
11 - "Compute the attention between an input query and the cached "
12 - "keys/values using PagedAttention.");
13 - ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
14 -
15 - // Activation ops
16 - ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
17 - ops.def("gelu_and_mul", &gelu_and_mul,
18 - "Activation function used in GeGLU with `none` approximation.");
19 - ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
20 - "Activation function used in GeGLU with `tanh` approximation.");
21 - ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
22 - ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
23 -
24 - // Layernorm
25 - ops.def("rms_norm", &rms_norm,
26 - "Apply Root Mean Square (RMS) Normalization to the input tensor.");
27 -
28 - ops.def("fused_add_rms_norm", &fused_add_rms_norm,
29 - "In-place fused Add and RMS Normalization");
30 -
31 - // Rotary embedding
32 - ops.def("rotary_embedding", &rotary_embedding,
33 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
34 -
35 - // Cache ops
36 - pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
37 - cache_ops.def("swap_blocks", &swap_blocks,
38 - "Swap in (out) the cache blocks from src to dst");
39 - cache_ops.def("copy_blocks", &copy_blocks,
40 - "Copy the cache blocks from src to dst");
41 - cache_ops.def("reshape_and_cache", &reshape_and_cache,
42 - "Reshape the key and value tensors and cache them");
43 -}
csrc/cpu/torch_bindings.cpp new
+106
@@ -0,0 +1,106 @@
1 +#include "cache.h"
2 +#include "ops.h"
3 +#include "registration.h"
4 +
5 +#include <torch/library.h>
6 +
7 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
8 + // vLLM custom ops
9 +
10 + // Attention ops
11 + // Compute the attention between an input query and the cached keys/values
12 + // using PagedAttention.
13 + ops.def(
14 + "paged_attention_v1("
15 + " Tensor! out, Tensor query, Tensor key_cache,"
16 + " Tensor value_cache, int num_kv_heads, float scale,"
17 + " Tensor block_tables, Tensor seq_lens, int block_size,"
18 + " int max_seq_len, Tensor? alibi_slopes,"
19 + " str kv_cache_dtype, float kv_scale, int tp_rank,"
20 + " int blocksparse_local_blocks,"
21 + " int blocksparse_vert_stride, int blocksparse_block_size,"
22 + " int blocksparse_head_sliding_step) -> ()");
23 + ops.impl("paged_attention_v1", torch::kCPU, &paged_attention_v1);
24 +
25 + // PagedAttention V2.
26 + ops.def(
27 + "paged_attention_v2("
28 + " Tensor! out, Tensor exp_sums, Tensor max_logits,"
29 + " Tensor tmp_out, Tensor query, Tensor key_cache,"
30 + " Tensor value_cache, int num_kv_heads, float scale,"
31 + " Tensor block_tables, Tensor seq_lens, int block_size,"
32 + " int max_seq_len, Tensor? alibi_slopes,"
33 + " str kv_cache_dtype, float kv_scale, int tp_rank,"
34 + " int blocksparse_local_blocks,"
35 + " int blocksparse_vert_stride, int blocksparse_block_size,"
36 + " int blocksparse_head_sliding_step) -> ()");
37 + ops.impl("paged_attention_v2", torch::kCPU, &paged_attention_v2);
38 +
39 + // Activation ops
40 +
41 + // Activation function used in SwiGLU.
42 + ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
43 + ops.impl("silu_and_mul", torch::kCPU, &silu_and_mul);
44 +
45 + // Activation function used in GeGLU with `none` approximation.
46 + ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
47 + ops.impl("gelu_and_mul", torch::kCPU, &gelu_and_mul);
48 +
49 + // Activation function used in GeGLU with `tanh` approximation.
50 + ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
51 + ops.impl("gelu_tanh_and_mul", torch::kCPU, &gelu_tanh_and_mul);
52 +
53 + // GELU implementation used in GPT-2.
54 + ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
55 + ops.impl("gelu_new", torch::kCPU, &gelu_new);
56 +
57 + // Approximate GELU implementation.
58 + ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
59 + ops.impl("gelu_fast", torch::kCPU, &gelu_fast);
60 +
61 + // Layernorm
62 + // Apply Root Mean Square (RMS) Normalization to the input tensor.
63 + ops.def(
64 + "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
65 + "()");
66 + ops.impl("rms_norm", torch::kCPU, &rms_norm);
67 +
68 + // In-place fused Add and RMS Normalization.
69 + ops.def(
70 + "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor weight, "
71 + "float epsilon) -> ()");
72 + ops.impl("fused_add_rms_norm", torch::kCPU, &fused_add_rms_norm);
73 +
74 + // Rotary embedding
75 + // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
76 + ops.def(
77 + "rotary_embedding(Tensor positions, Tensor! query,"
78 + " Tensor! key, int head_size,"
79 + " Tensor cos_sin_cache, bool is_neox) -> ()");
80 + ops.impl("rotary_embedding", torch::kCPU, &rotary_embedding);
81 +}
82 +
83 +TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
84 + // Cache ops
85 + // Swap in (out) the cache blocks from src to dst.
86 + cache_ops.def(
87 + "swap_blocks(Tensor src, Tensor! dst, Tensor block_mapping) -> ()");
88 + cache_ops.impl("swap_blocks", torch::kCPU, &swap_blocks);
89 +
90 + // Copy the cache blocks from src to dst.
91 + cache_ops.def(
92 + "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
93 + "block_mapping) -> ()");
94 + cache_ops.impl("copy_blocks", torch::kCPU, &copy_blocks);
95 +
96 + // Reshape the key and value tensors and cache them.
97 + cache_ops.def(
98 + "reshape_and_cache(Tensor key, Tensor value,"
99 + " Tensor! key_cache, Tensor! value_cache,"
100 + " Tensor slot_mapping,"
101 + " str kv_cache_dtype,"
102 + " float kv_scale) -> ()");
103 + cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
104 +}
105 +
106 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cuda_utils.h
+2 -4
@@ -1,7 +1,5 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +int64_t get_device_attribute(int64_t attribute, int64_t device_id);
4
5 -int get_device_attribute(int attribute, int device_id);
6 -
7 -int get_max_shared_memory_per_block_device_attribute(int device_id);
5 +int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id);
csrc/cuda_utils_kernels.cu
+3 -3
@@ -2,7 +2,7 @@
2 #include <hip/hip_runtime.h>
3 #include <hip/hip_runtime_api.h>
4 #endif
5 -int get_device_attribute(int attribute, int device_id) {
5 +int64_t get_device_attribute(int64_t attribute, int64_t device_id) {
6 int device, value;
7 if (device_id < 0) {
8 cudaGetDevice(&device);
@@ -14,8 +14,8 @@ int get_device_attribute(int attribute, int device_id) {
14 return value;
15 }
16
17 -int get_max_shared_memory_per_block_device_attribute(int device_id) {
18 - int attribute;
17 +int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id) {
18 + int64_t attribute;
19 // https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
20 // cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
21
csrc/custom_all_reduce.cu
+14 -8
@@ -1,17 +1,17 @@
1 #include <ATen/cuda/Exceptions.h>
2 #include <c10/cuda/CUDAGuard.h>
3 #include <c10/cuda/CUDAStream.h>
4 -#include <torch/extension.h>
4 +#include <torch/all.h>
5
6 #include "custom_all_reduce.cuh"
7
8 -// fake pointer type
9 -using fptr_t = uint64_t;
8 +// fake pointer type, must match fptr_t type in ops.h
9 +using fptr_t = int64_t;
10 static_assert(sizeof(void*) == sizeof(fptr_t));
11
12 fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13 const std::vector<std::string>& handles,
14 - const std::vector<int64_t>& offsets, int rank,
14 + const std::vector<int64_t>& offsets, int64_t rank,
15 bool full_nvlink) {
16 int world_size = offsets.size();
17 if (world_size > 8)
@@ -55,7 +55,7 @@ bool _is_weak_contiguous(torch::Tensor& t) {
55 t.numel() * t.element_size());
56 }
57
58 -bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
58 +bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
59 bool full_nvlink) {
60 auto inp_size = inp.numel() * inp.element_size();
61 // custom allreduce requires input byte size to be multiples of 16
@@ -125,7 +125,7 @@ void dispose(fptr_t _fa) {
125 delete fa;
126 }
127
128 -int meta_size() { return sizeof(vllm::Signal); }
128 +int64_t meta_size() { return sizeof(vllm::Signal); }
129
130 void register_buffer(fptr_t _fa, torch::Tensor& t,
131 const std::vector<std::string>& handles,
@@ -134,10 +134,16 @@ void register_buffer(fptr_t _fa, torch::Tensor& t,
134 fa->register_buffer(handles, offsets, t.data_ptr());
135 }
136
137 -std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
137 +std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
138 fptr_t _fa) {
139 auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140 - return fa->get_graph_buffer_ipc_meta();
140 + auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta();
141 + auto options =
142 + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU);
143 + auto handles =
144 + torch::empty({static_cast<int64_t>(handle_bytes.size())}, options);
145 + std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size());
146 + return {handles, std::move(offsets)};
147 }
148
149 void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
csrc/dispatch_utils.h
+1 -1
@@ -4,7 +4,7 @@
4 */
5 #pragma once
6
7 -#include <torch/extension.h>
7 +#include <torch/all.h>
8
9 #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10 AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
csrc/layernorm_kernels.cu
+3 -3
@@ -1,4 +1,4 @@
1 -#include <torch/extension.h>
1 +#include <torch/all.h>
2 #include <ATen/cuda/CUDAContext.h>
3 #include <c10/cuda/CUDAGuard.h>
4
@@ -291,7 +291,7 @@ fused_add_rms_norm_kernel(
291 void rms_norm(torch::Tensor& out, // [..., hidden_size]
292 torch::Tensor& input, // [..., hidden_size]
293 torch::Tensor& weight, // [hidden_size]
294 - float epsilon) {
294 + double epsilon) {
295 int hidden_size = input.size(-1);
296 int num_tokens = input.numel() / hidden_size;
297
@@ -319,7 +319,7 @@ void rms_norm(torch::Tensor& out, // [..., hidden_size]
319 void fused_add_rms_norm(torch::Tensor& input, // [..., hidden_size]
320 torch::Tensor& residual, // [..., hidden_size]
321 torch::Tensor& weight, // [hidden_size]
322 - float epsilon) {
322 + double epsilon) {
323 int hidden_size = input.size(-1);
324 int num_tokens = input.numel() / hidden_size;
325
csrc/moe/moe_ops.cpp deleted
-8
@@ -1,8 +0,0 @@
1 -#include "moe_ops.h"
2 -
3 -#include <torch/extension.h>
4 -
5 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6 - m.def("topk_softmax", &topk_softmax,
7 - "Apply topk softmax to the gating outputs.");
8 -}
csrc/moe/moe_ops.h
+1 -1
@@ -1,6 +1,6 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +#include <torch/all.h>
4
5 void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6 torch::Tensor& token_expert_indices,
csrc/moe/topk_softmax_kernels.cu
+1 -1
@@ -16,7 +16,7 @@
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 -#include <torch/extension.h>
19 +#include <torch/all.h>
20 #include <ATen/cuda/CUDAContext.h>
21 #include <c10/cuda/CUDAGuard.h>
22 #include "../cuda_compat.h"
csrc/moe/torch_bindings.cpp new
+12
@@ -0,0 +1,12 @@
1 +#include "registration.h"
2 +#include "moe_ops.h"
3 +
4 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5 + // Apply topk softmax to the gating outputs.
6 + m.def(
7 + "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
8 + "token_expert_indices, Tensor gating_output) -> ()");
9 + m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
10 +}
11 +
12 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/moe_align_block_size_kernels.cu
+3 -3
@@ -1,4 +1,4 @@
1 -#include <torch/extension.h>
1 +#include <torch/all.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 #include <ATen/ATen.h>
@@ -108,8 +108,8 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
108 }
109 } // namespace vllm
110
111 -void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
112 - int block_size, torch::Tensor sorted_token_ids,
111 +void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
112 + int64_t block_size, torch::Tensor sorted_token_ids,
113 torch::Tensor experts_ids,
114 torch::Tensor num_tokens_post_pad) {
115 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
csrc/ops.h
+35 -33
@@ -1,40 +1,42 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +#include <torch/library.h>
4
5 void paged_attention_v1(
6 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
7 - torch::Tensor& value_cache, int num_kv_heads, float scale,
8 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
9 - int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
11 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
12 - const int blocksparse_block_size, const int blocksparse_head_sliding_step);
7 + torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
8 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
9 + int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
11 + const int64_t blocksparse_local_blocks,
12 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
13 + const int64_t blocksparse_head_sliding_step);
14
15 void paged_attention_v2(
16 torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
17 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
17 - torch::Tensor& value_cache, int num_kv_heads, float scale,
18 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
19 - int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
20 - const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
21 - const int blocksparse_local_blocks, const int blocksparse_vert_stride,
22 - const int blocksparse_block_size, const int blocksparse_head_sliding_step);
18 + torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
19 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
20 + int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
21 + const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
22 + const int64_t blocksparse_local_blocks,
23 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
24 + const int64_t blocksparse_head_sliding_step);
25
26 void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
25 - float epsilon);
27 + double epsilon);
28
29 void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
28 - torch::Tensor& weight, float epsilon);
30 + torch::Tensor& weight, double epsilon);
31
32 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
31 - torch::Tensor& key, int head_size,
33 + torch::Tensor& key, int64_t head_size,
34 torch::Tensor& cos_sin_cache, bool is_neox);
35
36 void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
35 - torch::Tensor& key, int head_size,
37 + torch::Tensor& key, int64_t head_size,
38 torch::Tensor& cos_sin_cache, bool is_neox,
37 - int rot_dim,
39 + int64_t rot_dim,
40 torch::Tensor& cos_sin_cache_offsets);
41
42 void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
@@ -60,12 +62,12 @@ torch::Tensor aqlm_dequant(const torch::Tensor& codes,
62
63 torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
64 torch::Tensor _scaling_factors, torch::Tensor _zeros,
63 - int split_k_iters);
65 + int64_t split_k_iters);
66
67 torch::Tensor awq_dequantize(torch::Tensor _kernel,
68 torch::Tensor _scaling_factors,
67 - torch::Tensor _zeros, int split_k_iters, int thx,
68 - int thy);
69 + torch::Tensor _zeros, int64_t split_k_iters,
70 + int64_t thx, int64_t thy);
71
72 torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
73 torch::Tensor& b_scales, torch::Tensor& workspace,
@@ -88,9 +90,9 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
90 int64_t size_k, int64_t size_n,
91 int64_t num_bits);
92
91 -int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
92 - torch::Tensor const& b, torch::Tensor const& a_scales,
93 - torch::Tensor const& b_scales);
93 +void cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
94 + torch::Tensor const& b, torch::Tensor const& a_scales,
95 + torch::Tensor const& b_scales);
96
97 #endif
98
@@ -106,9 +108,9 @@ void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
108 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
109 torch::Tensor b_gptq_qzeros,
110 torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
109 - bool use_exllama, int bit);
111 + bool use_exllama, int64_t bit);
112
111 -void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit);
113 +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
114
115 void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
116 torch::Tensor& scale);
@@ -116,28 +118,28 @@ void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
118 void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
119 torch::Tensor& scale);
120
119 -void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
120 - int block_size, torch::Tensor sorted_token_ids,
121 +void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
122 + int64_t block_size, torch::Tensor sorted_token_ids,
123 torch::Tensor experts_ids,
124 torch::Tensor num_tokens_post_pad);
125
126 #ifndef USE_ROCM
125 -using fptr_t = uint64_t;
127 +using fptr_t = int64_t;
128 fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
129 const std::vector<std::string>& handles,
128 - const std::vector<int64_t>& offsets, int rank,
130 + const std::vector<int64_t>& offsets, int64_t rank,
131 bool full_nvlink);
130 -bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
132 +bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
133 bool full_nvlink);
134 void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
135 void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
136 torch::Tensor& out);
137 void dispose(fptr_t _fa);
136 -int meta_size();
138 +int64_t meta_size();
139 void register_buffer(fptr_t _fa, torch::Tensor& t,
140 const std::vector<std::string>& handles,
141 const std::vector<int64_t>& offsets);
140 -std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
142 +std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
143 fptr_t _fa);
144 void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
145 const std::vector<std::vector<int64_t>>& offsets);
csrc/pos_encoding_kernels.cu
+6 -6
@@ -1,4 +1,4 @@
1 -#include <torch/extension.h>
1 +#include <torch/all.h>
2 #include <ATen/cuda/CUDAContext.h>
3 #include <c10/cuda/CUDAGuard.h>
4
@@ -127,7 +127,7 @@ void rotary_embedding(
127 // [num_tokens, num_heads * head_size]
128 torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
129 // [num_tokens, num_kv_heads * head_size]
130 - int head_size,
130 + int64_t head_size,
131 torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
132 bool is_neox) {
133 int64_t num_tokens = query.numel() / query.size(-1);
@@ -138,7 +138,7 @@ void rotary_embedding(
138 int64_t key_stride = key.stride(-2);
139
140 dim3 grid(num_tokens);
141 - dim3 block(std::min(num_heads * rot_dim / 2, 512));
141 + dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
142 const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
143 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
144 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
@@ -168,9 +168,9 @@ void batched_rotary_embedding(
168 // [num_tokens, num_heads * head_size]
169 torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
170 // [num_tokens, num_kv_heads * head_size]
171 - int head_size,
171 + int64_t head_size,
172 torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
173 - bool is_neox, int rot_dim,
173 + bool is_neox, int64_t rot_dim,
174 torch::Tensor& cos_sin_cache_offsets // [num_tokens]
175 ) {
176 int64_t num_tokens = cos_sin_cache_offsets.size(0);
@@ -180,7 +180,7 @@ void batched_rotary_embedding(
180 int64_t key_stride = key.stride(-2);
181
182 dim3 grid(num_tokens);
183 - dim3 block(std::min(num_heads * rot_dim / 2, 512));
183 + dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
184 const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
185 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
186 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
csrc/punica/punica_ops.cu
+3 -3
@@ -1,4 +1,4 @@
1 -#include <torch/extension.h>
1 +#include <torch/all.h>
2 #include <c10/cuda/CUDAGuard.h>
3 #include <cstdint>
4
@@ -88,7 +88,7 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
88 }
89
90 void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91 - torch::Tensor indicies, int64_t layer_idx, float scale) {
91 + torch::Tensor indicies, int64_t layer_idx, double scale) {
92 CHECK_INPUT(y);
93 CHECK_INPUT(x);
94 CHECK_INPUT(w);
@@ -320,7 +320,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
320
321 void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
322 torch::Tensor indicies, int64_t layer_idx,
323 - float scale, int64_t h_in, int64_t h_out,
323 + double scale, int64_t h_in, int64_t h_out,
324 int64_t y_offset) {
325 CHECK_INPUT(y);
326 CHECK_INPUT(x);
csrc/punica/punica_ops.h
+3 -3
@@ -1,11 +1,11 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +#include <torch/all.h>
4
5 void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6 - torch::Tensor indicies, int64_t layer_idx, float scale);
6 + torch::Tensor indicies, int64_t layer_idx, double scale);
7
8 void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9 torch::Tensor indicies, int64_t layer_idx,
10 - float scale, int64_t h_in, int64_t h_out,
10 + double scale, int64_t h_in, int64_t h_out,
11 int64_t y_offset);
csrc/punica/punica_pybind.cpp deleted
-13
@@ -1,13 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -#include "punica_ops.h"
4 -
5 -//====== pybind ======
6 -
7 -#define DEFINE_pybind(name) m.def(#name, &name, #name);
8 -
9 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
10 - m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
11 - m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
12 - "dispatch_bgmv_low_level");
13 -}
csrc/punica/torch_bindings.cpp new
+18
@@ -0,0 +1,18 @@
1 +#include "registration.h"
2 +#include "punica_ops.h"
3 +
4 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5 + m.def(
6 + "dispatch_bgmv(Tensor! y, Tensor x, Tensor w, Tensor indicies, int "
7 + "layer_idx, float scale) -> ()");
8 + m.impl("dispatch_bgmv", torch::kCUDA, &dispatch_bgmv);
9 +
10 + m.def(
11 + "dispatch_bgmv_low_level(Tensor! y, Tensor x, Tensor w,"
12 + "Tensor indicies, int layer_idx,"
13 + "float scale, int h_in, int h_out,"
14 + "int y_offset) -> ()");
15 + m.impl("dispatch_bgmv_low_level", torch::kCUDA, &dispatch_bgmv_low_level);
16 +}
17 +
18 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/pybind.cpp deleted
-114
@@ -1,114 +0,0 @@
1 -#include "cache.h"
2 -#include "cuda_utils.h"
3 -#include "ops.h"
4 -#include <torch/extension.h>
5 -
6 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
7 - // vLLM custom ops
8 - pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9 -
10 - // Attention ops
11 - ops.def("paged_attention_v1", &paged_attention_v1,
12 - "Compute the attention between an input query and the cached "
13 - "keys/values using PagedAttention.");
14 - ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15 -
16 - // Activation ops
17 - ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18 - ops.def("gelu_and_mul", &gelu_and_mul,
19 - "Activation function used in GeGLU with `none` approximation.");
20 - ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21 - "Activation function used in GeGLU with `tanh` approximation.");
22 - ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23 - ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24 -
25 - // Layernorm
26 - ops.def("rms_norm", &rms_norm,
27 - "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28 -
29 - ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30 - "In-place fused Add and RMS Normalization");
31 -
32 - // Rotary embedding
33 - ops.def("rotary_embedding", &rotary_embedding,
34 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35 -
36 - ops.def("batched_rotary_embedding", &batched_rotary_embedding,
37 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key "
38 - "(supports multiple loras)");
39 -
40 -// Quantization ops
41 -#ifndef USE_ROCM
42 - ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
43 - ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
44 - ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
45 - ops.def("marlin_gemm", &marlin_gemm,
46 - "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
47 - ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm,
48 - "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
49 - ops.def("gptq_marlin_gemm", &gptq_marlin_gemm,
50 - "gptq_marlin Optimized Quantized GEMM for GPTQ");
51 - ops.def("gptq_marlin_repack", &gptq_marlin_repack,
52 - "gptq_marlin repack from GPTQ");
53 - ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
54 - ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq,
55 - "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or "
56 - "per-row/column quantization.");
57 -#endif
58 -
59 - ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
60 - ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
61 - ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
62 - ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant,
63 - "Compute FP8 quantized tensor for given scaling factor");
64 - ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant,
65 - "Compute FP8 quantized tensor and scaling factor");
66 - ops.def("moe_align_block_size", &moe_align_block_size,
67 - "Aligning the number of tokens to be processed by each expert such "
68 - "that it is divisible by the block size.");
69 -
70 - ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71 - "Compute int8 quantized tensor for given scaling factor");
72 -
73 - ops.def("dynamic_scaled_int8_quant", &dynamic_scaled_int8_quant,
74 - "Compute int8 quantized tensor and scaling factor");
75 -
76 - // Cache ops
77 - pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
78 - cache_ops.def("swap_blocks", &swap_blocks,
79 - "Swap in (out) the cache blocks from src to dst");
80 - cache_ops.def("copy_blocks", &copy_blocks,
81 - "Copy the cache blocks from src to dst");
82 - cache_ops.def("reshape_and_cache", &reshape_and_cache,
83 - "Reshape the key and value tensors and cache them");
84 - cache_ops.def("reshape_and_cache_flash", &reshape_and_cache_flash,
85 - "Reshape the key and value tensors and cache them");
86 - cache_ops.def("convert_fp8", &convert_fp8,
87 - "Convert the key and value cache to fp8 data type");
88 -
89 - // Cuda utils
90 - pybind11::module cuda_utils =
91 - m.def_submodule("cuda_utils", "vLLM cuda utils");
92 - cuda_utils.def("get_device_attribute", &get_device_attribute,
93 - "Gets the specified device attribute.");
94 -
95 - cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
96 - &get_max_shared_memory_per_block_device_attribute,
97 - "Gets the maximum shared memory per block device attribute.");
98 -
99 -#ifndef USE_ROCM
100 - // Custom all-reduce kernels
101 - pybind11::module custom_ar = m.def_submodule("custom_ar", "custom allreduce");
102 - custom_ar.def("init_custom_ar", &init_custom_ar, "init_custom_ar");
103 - custom_ar.def("should_custom_ar", &should_custom_ar, "should_custom_ar");
104 - custom_ar.def("all_reduce_reg", &all_reduce_reg, "all_reduce_reg");
105 - custom_ar.def("all_reduce_unreg", &all_reduce_unreg, "all_reduce_unreg");
106 - custom_ar.def("dispose", &dispose, "dispose");
107 - custom_ar.def("meta_size", &meta_size, "meta_size");
108 - custom_ar.def("register_buffer", &register_buffer, "register_buffer");
109 - custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta,
110 - "get_graph_buffer_ipc_meta");
111 - custom_ar.def("register_graph_buffers", &register_graph_buffers,
112 - "register_graph_buffers");
113 -#endif
114 -}
csrc/quantization/aqlm/gemm_kernels.cu
+1 -1
@@ -18,7 +18,7 @@
18 #include <cuda.h>
19 #include <cuda_fp16.h>
20 #include <cuda_runtime.h>
21 -#include <torch/extension.h>
21 +#include <torch/all.h>
22 #include <c10/cuda/CUDAStream.h>
23 #include <c10/cuda/CUDAGuard.h>
24
csrc/quantization/awq/gemm_kernels.cu
+4 -4
@@ -7,7 +7,7 @@ Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
7 }
8 */
9
10 -#include <torch/extension.h>
10 +#include <torch/all.h>
11 #include <c10/cuda/CUDAGuard.h>
12
13 #include "dequantize.cuh"
@@ -435,8 +435,8 @@ __global__ void __launch_bounds__(64)
435
436 torch::Tensor awq_dequantize(torch::Tensor _kernel,
437 torch::Tensor _scaling_factors,
438 - torch::Tensor _zeros, int split_k_iters, int thx,
439 - int thy) {
438 + torch::Tensor _zeros, int64_t split_k_iters,
439 + int64_t thx, int64_t thy) {
440 int in_c = _kernel.size(0);
441 int qout_c = _kernel.size(1);
442 int out_c = qout_c * 8;
@@ -491,7 +491,7 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
491
492 torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
493 torch::Tensor _scaling_factors, torch::Tensor _zeros,
494 - int split_k_iters) {
494 + int64_t split_k_iters) {
495 int num_in_feats = _in_feats.size(0);
496 int num_in_channels = _in_feats.size(1);
497 const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+1 -1
@@ -1,5 +1,5 @@
1 #include <ATen/cuda/CUDAContext.h>
2 -#include <torch/extension.h>
2 +#include <torch/all.h>
3 #include <cmath>
4
5 #include "../../dispatch_utils.h"
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+1 -1
@@ -1,5 +1,5 @@
1 #include <stddef.h>
2 -#include <torch/extension.h>
2 +#include <torch/all.h>
3
4 #include <ATen/cuda/CUDAContext.h>
5
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+1 -1
@@ -4,7 +4,7 @@
4
5 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
6
7 -#include <torch/extension.h>
7 +#include <torch/all.h>
8
9 #include <ATen/cuda/CUDAContext.h>
10
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+1 -1
@@ -1,7 +1,7 @@
1 #include <cudaTypedefs.h>
2
3 #include <c10/cuda/CUDAGuard.h>
4 -#include <torch/extension.h>
4 +#include <torch/all.h>
5
6 void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
7 torch::Tensor const& b,
csrc/quantization/fp8/common.cu
+1 -1
@@ -1,5 +1,5 @@
1 #include <ATen/cuda/CUDAContext.h>
2 -#include <torch/extension.h>
2 +#include <torch/all.h>
3 #include <c10/cuda/CUDAGuard.h>
4
5 #include <cmath>
csrc/quantization/gptq/q_gemm.cu
+3 -3
@@ -6,7 +6,7 @@ https://github.com/qwopqwop200/GPTQ-for-LLaMa
6 #include <cstdint>
7 #include <cstdio>
8
9 -#include <torch/extension.h>
9 +#include <torch/all.h>
10 #include <c10/cuda/CUDAGuard.h>
11 #include <ATen/cuda/CUDAContext.h>
12 #include <cuda_runtime.h>
@@ -1823,7 +1823,7 @@ void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
1823 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1824 torch::Tensor b_gptq_qzeros,
1825 torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
1826 - bool use_exllama, int bit) {
1826 + bool use_exllama, int64_t bit) {
1827 const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1828 auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1829 at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
@@ -1845,7 +1845,7 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1845 return c;
1846 }
1847
1848 -void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit) {
1848 +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit) {
1849 const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
1850 vllm::gptq::shuffle_exllama_weight(
1851 (uint32_t*)q_weight.data_ptr(),
csrc/quantization/gptq_marlin/gptq_marlin.cu
+1 -1
@@ -1867,4 +1867,4 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1867 return c;
1868 }
1869
1870 -#endif
\ No newline at end of file
1870 +#endif
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+1 -1
@@ -1,6 +1,6 @@
1 #pragma once
2
3 -#include <torch/extension.h>
3 +#include <torch/all.h>
4
5 #include <ATen/cuda/CUDAContext.h>
6 #include <c10/cuda/CUDAGuard.h>
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+1 -1
@@ -15,7 +15,7 @@
15 * limitations under the License.
16 */
17
18 -#include <torch/extension.h>
18 +#include <torch/all.h>
19
20 #include <ATen/cuda/CUDAContext.h>
21 #include <c10/cuda/CUDAGuard.h>
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+1 -1
@@ -16,7 +16,7 @@
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 -#include <torch/extension.h>
19 +#include <torch/all.h>
20
21 #include <ATen/cuda/CUDAContext.h>
22 #include <c10/cuda/CUDAGuard.h>
csrc/quantization/squeezellm/quant_cuda_kernel.cu
-1
@@ -1,5 +1,4 @@
1 #include <torch/all.h>
2 -#include <torch/python.h>
2 #include <cuda.h>
3 #include <cuda_runtime.h>
4 #include <cuda_fp16.h>
csrc/registration.h new
+22
@@ -0,0 +1,22 @@
1 +#pragma once
2 +
3 +#include <Python.h>
4 +
5 +#define _CONCAT(A, B) A##B
6 +#define CONCAT(A, B) _CONCAT(A, B)
7 +
8 +#define _STRINGIFY(A) #A
9 +#define STRINGIFY(A) _STRINGIFY(A)
10 +
11 +// A version of the TORCH_LIBRARY macro that expands the NAME, i.e. so NAME
12 +// could be a macro instead of a literal token.
13 +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE)
14 +
15 +// REGISTER_EXTENSION allows the shared library to be loaded and initialized
16 +// via python's import statement.
17 +#define REGISTER_EXTENSION(NAME) \
18 + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \
19 + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, \
20 + STRINGIFY(NAME), nullptr, 0, nullptr}; \
21 + return PyModule_Create(&module); \
22 + }
csrc/torch_bindings.cpp new
+283
@@ -0,0 +1,283 @@
1 +#include "cache.h"
2 +#include "cuda_utils.h"
3 +#include "ops.h"
4 +#include "registration.h"
5 +
6 +#include <torch/library.h>
7 +
8 +// Note on op signatures:
9 +// The X_meta signatures are for the meta functions corresponding to op X.
10 +// They must be kept in sync with the signature for X. Generally, only
11 +// functions that return Tensors require a meta function.
12 +//
13 +// See the following links for detailed docs on op registration and function
14 +// schemas.
15 +// https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ptttacy8y1u9
16 +// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#annotations
17 +
18 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
19 + // vLLM custom ops
20 +
21 + // Attention ops
22 + // Compute the attention between an input query and the cached
23 + // keys/values using PagedAttention.
24 + ops.def(
25 + "paged_attention_v1("
26 + " Tensor! out, Tensor query, Tensor key_cache,"
27 + " Tensor value_cache, int num_kv_heads, float scale,"
28 + " Tensor block_tables, Tensor seq_lens, int block_size,"
29 + " int max_seq_len, Tensor? alibi_slopes,"
30 + " str kv_cache_dtype, float kv_scale, int tp_rank,"
31 + " int blocksparse_local_blocks,"
32 + " int blocksparse_vert_stride, int blocksparse_block_size,"
33 + " int blocksparse_head_sliding_step) -> ()");
34 + ops.impl("paged_attention_v1", torch::kCUDA, &paged_attention_v1);
35 +
36 + // PagedAttention V2.
37 + ops.def(
38 + "paged_attention_v2("
39 + " Tensor! out, Tensor exp_sums, Tensor max_logits,"
40 + " Tensor tmp_out, Tensor query, Tensor key_cache,"
41 + " Tensor value_cache, int num_kv_heads, float scale,"
42 + " Tensor block_tables, Tensor seq_lens, int block_size,"
43 + " int max_seq_len, Tensor? alibi_slopes,"
44 + " str kv_cache_dtype, float kv_scale, int tp_rank,"
45 + " int blocksparse_local_blocks,"
46 + " int blocksparse_vert_stride, int blocksparse_block_size,"
47 + " int blocksparse_head_sliding_step) -> ()");
48 + ops.impl("paged_attention_v2", torch::kCUDA, &paged_attention_v2);
49 +
50 + // Activation ops
51 + // Activation function used in SwiGLU.
52 + ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
53 + ops.impl("silu_and_mul", torch::kCUDA, &silu_and_mul);
54 +
55 + // Activation function used in GeGLU with `none` approximation.
56 + ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
57 + ops.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul);
58 +
59 + // Activation function used in GeGLU with `tanh` approximation.
60 + ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
61 + ops.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul);
62 +
63 + // GELU implementation used in GPT-2.
64 + ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
65 + ops.impl("gelu_new", torch::kCUDA, &gelu_new);
66 +
67 + // Approximate GELU implementation.
68 + ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
69 + ops.impl("gelu_fast", torch::kCUDA, &gelu_fast);
70 +
71 + // Layernorm
72 + // Apply Root Mean Square (RMS) Normalization to the input tensor.
73 + ops.def(
74 + "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
75 + "()");
76 + ops.impl("rms_norm", torch::kCUDA, &rms_norm);
77 +
78 + // In-place fused Add and RMS Normalization.
79 + ops.def(
80 + "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor weight, "
81 + "float epsilon) -> ()");
82 + ops.impl("fused_add_rms_norm", torch::kCUDA, &fused_add_rms_norm);
83 +
84 + // Rotary embedding
85 + // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
86 + ops.def(
87 + "rotary_embedding(Tensor positions, Tensor! query,"
88 + " Tensor! key, int head_size,"
89 + " Tensor cos_sin_cache, bool is_neox) -> ()");
90 + ops.impl("rotary_embedding", torch::kCUDA, &rotary_embedding);
91 +
92 + // Apply GPT-NeoX or GPT-J style rotary embedding to query and key
93 + // (supports multiple loras).
94 + ops.def(
95 + "batched_rotary_embedding(Tensor positions, Tensor! query,"
96 + " Tensor! key, int head_size,"
97 + " Tensor cos_sin_cache, bool is_neox,"
98 + " int rot_dim,"
99 + " Tensor cos_sin_cache_offsets) -> ()");
100 + ops.impl("batched_rotary_embedding", torch::kCUDA, &batched_rotary_embedding);
101 +
102 + // Quantization ops
103 +#ifndef USE_ROCM
104 + // Quantized GEMM for AQLM.
105 + ops.def("aqlm_gemm", &aqlm_gemm);
106 + ops.impl("aqlm_gemm", torch::kCUDA, &aqlm_gemm);
107 +
108 + // Decompression method for AQLM.
109 + ops.def("aqlm_dequant", &aqlm_dequant);
110 + ops.impl("aqlm_dequant", torch::kCUDA, &aqlm_dequant);
111 +
112 + // Quantized GEMM for AWQ.
113 + ops.def("awq_gemm", &awq_gemm);
114 + ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
115 +
116 + // Dequantization for AWQ.
117 + ops.def("awq_dequantize", &awq_dequantize);
118 + ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
119 +
120 + // Marlin (Dense) Optimized Quantized GEMM for GPTQ.
121 + ops.def("marlin_gemm", &marlin_gemm);
122 + ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
123 +
124 + // Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
125 + ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
126 + ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
127 +
128 + // gptq_marlin Optimized Quantized GEMM for GPTQ.
129 + ops.def("gptq_marlin_gemm", &gptq_marlin_gemm);
130 + ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
131 +
132 + // gptq_marlin repack from GPTQ.
133 + ops.def("gptq_marlin_repack", &gptq_marlin_repack);
134 + ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
135 +
136 + // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
137 + // quantization.
138 + ops.def(
139 + "cutlass_scaled_mm_dq(Tensor! out, Tensor a,"
140 + " Tensor b, Tensor a_scales,"
141 + " Tensor b_scales) -> ()");
142 + ops.impl("cutlass_scaled_mm_dq", torch::kCUDA, &cutlass_scaled_mm_dq);
143 +#endif
144 +
145 + // Quantized GEMM for GPTQ.
146 + ops.def("gptq_gemm", &gptq_gemm);
147 + ops.impl("gptq_gemm", torch::kCUDA, &gptq_gemm);
148 +
149 + // Post processing for GPTQ.
150 + ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
151 + ops.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle);
152 +
153 + // Quantized GEMM for SqueezeLLM.
154 + ops.def(
155 + "squeezellm_gemm(Tensor vec, Tensor mat, Tensor! mul, Tensor "
156 + "lookup_table) -> ()");
157 + ops.impl("squeezellm_gemm", torch::kCUDA, &squeezellm_gemm);
158 +
159 + // Compute FP8 quantized tensor for given scaling factor.
160 + ops.def(
161 + "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
162 + ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
163 +
164 + // Compute FP8 quantized tensor and scaling factor.
165 + ops.def(
166 + "dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
167 + "()");
168 + ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
169 +
170 + // Aligning the number of tokens to be processed by each expert such
171 + // that it is divisible by the block size.
172 + ops.def(
173 + "moe_align_block_size(Tensor topk_ids, int num_experts,"
174 + " int block_size, Tensor! sorted_token_ids,"
175 + " Tensor! experts_ids,"
176 + " Tensor! num_tokens_post_pad) -> ()");
177 + ops.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
178 +
179 + // Compute int8 quantized tensor for given scaling factor.
180 + ops.def(
181 + "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
182 + "()");
183 + ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
184 +
185 + // Compute int8 quantized tensor and scaling factor
186 + ops.def(
187 + "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
188 + "()");
189 + ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
190 + &dynamic_scaled_int8_quant);
191 +}
192 +
193 +TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
194 + // Cache ops
195 + // Swap in (out) the cache blocks from src to dst.
196 + cache_ops.def(
197 + "swap_blocks(Tensor src, Tensor! dst, Tensor block_mapping) -> ()");
198 + cache_ops.impl("swap_blocks", torch::kCUDA, &swap_blocks);
199 +
200 + // Copy the cache blocks from src to dst.
201 + cache_ops.def(
202 + "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
203 + "block_mapping) -> ()");
204 + cache_ops.impl("copy_blocks", torch::kCUDA, &copy_blocks);
205 +
206 + // Reshape the key and value tensors and cache them.
207 + cache_ops.def(
208 + "reshape_and_cache(Tensor key, Tensor value,"
209 + " Tensor! key_cache, Tensor! value_cache,"
210 + " Tensor slot_mapping,"
211 + " str kv_cache_dtype,"
212 + " float kv_scale) -> ()");
213 + cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
214 +
215 + // Reshape the key and value tensors and cache them.
216 + cache_ops.def(
217 + "reshape_and_cache_flash(Tensor key, Tensor value,"
218 + " Tensor! key_cache,"
219 + " Tensor! value_cache,"
220 + " Tensor slot_mapping,"
221 + " str kv_cache_dtype) -> ()");
222 + cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
223 + &reshape_and_cache_flash);
224 +
225 + // Convert the key and value cache to fp8 data type.
226 + cache_ops.def(
227 + "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, str "
228 + "kv_cache_dtype) -> ()");
229 + cache_ops.impl("convert_fp8", torch::kCUDA, &convert_fp8);
230 +}
231 +
232 +TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
233 + // Cuda utils
234 +
235 + // Gets the specified device attribute.
236 + cuda_utils.def("get_device_attribute", &get_device_attribute);
237 + cuda_utils.impl("get_device_attribute", torch::kCUDA, &get_device_attribute);
238 +
239 + // Gets the maximum shared memory per block device attribute.
240 + cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
241 + &get_max_shared_memory_per_block_device_attribute);
242 + cuda_utils.impl("get_max_shared_memory_per_block_device_attribute",
243 + torch::kCUDA,
244 + &get_max_shared_memory_per_block_device_attribute);
245 +}
246 +
247 +#ifndef USE_ROCM
248 +TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
249 + // Custom all-reduce kernels
250 + custom_ar.def("init_custom_ar", &init_custom_ar);
251 + custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
252 +
253 + custom_ar.def("should_custom_ar", &should_custom_ar);
254 + custom_ar.impl("should_custom_ar", torch::kCUDA, &should_custom_ar);
255 +
256 + custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
257 + custom_ar.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg);
258 +
259 + custom_ar.def(
260 + "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> "
261 + "()");
262 + custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
263 +
264 + custom_ar.def("dispose", &dispose);
265 + custom_ar.impl("dispose", torch::kCPU, &dispose);
266 +
267 + custom_ar.def("meta_size", &meta_size);
268 + custom_ar.impl("meta_size", torch::kCPU, &meta_size);
269 +
270 + custom_ar.def("register_buffer", &register_buffer);
271 + custom_ar.impl("register_buffer", torch::kCUDA, &register_buffer);
272 +
273 + custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
274 + custom_ar.impl("get_graph_buffer_ipc_meta", torch::kCPU,
275 + &get_graph_buffer_ipc_meta);
276 +
277 + custom_ar.def("register_graph_buffers", &register_graph_buffers);
278 + custom_ar.impl("register_graph_buffers", torch::kCPU,
279 + &register_graph_buffers);
280 +}
281 +#endif
282 +
283 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
setup.py
+1 -1
@@ -60,7 +60,7 @@ def remove_prefix(text, prefix):
60 class CMakeExtension(Extension):
61
62 def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
63 - super().__init__(name, sources=[], **kwa)
63 + super().__init__(name, sources=[], py_limited_api=True, **kwa)
64 self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
65
66
tests/kernels/test_int8_quant.py
+4 -3
@@ -1,7 +1,8 @@
1 import pytest
2 import torch
3
4 -from vllm._C import ops
4 +# ruff: noqa: F401
5 +import vllm._C
6
7 DTYPES = [torch.half, torch.bfloat16, torch.float]
8 HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -33,7 +34,7 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
34
35 ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
36 scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
36 - ops.dynamic_scaled_int8_quant(ops_out, x, scales_out)
37 + torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out)
38
39 assert torch.allclose(scales_out, scales)
40 assert torch.allclose(torch_out, ops_out,
@@ -60,6 +61,6 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
61 out2 = torch.empty_like(x, dtype=torch.int8)
62 scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
63
63 - ops.static_scaled_int8_quant(out2, x, scale_argument)
64 + torch.ops._C.static_scaled_int8_quant(out2, x, scale_argument)
65 assert torch.allclose(out1, out2,
66 atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+164 -53
@@ -1,35 +1,47 @@
1 -from typing import Optional, Tuple, Type
1 +import contextlib
2 +from typing import List, Optional, Tuple, Type
3
4 import torch
5
6 try:
6 - from vllm._C import cache_ops as vllm_cache_ops
7 - from vllm._C import ops as vllm_ops
7 + import vllm._C
8 except ImportError as e:
9 from vllm.logger import init_logger
10 logger = init_logger(__name__)
11 logger.warning("Failed to import from vllm._C with %r", e)
12
13 +with contextlib.suppress(ImportError):
14 + import vllm._moe_C
15 +
16 +with contextlib.suppress(ImportError):
17 + # ruff: noqa: F401
18 + import vllm._punica_C
19 +
20 +
21 +def is_custom_op_supported(op_name: str) -> bool:
22 + op, overloads = torch._C._jit_get_operation(op_name)
23 + return op is not None
24 +
25
26 # activation ops
27 def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
16 - vllm_ops.silu_and_mul(out, x)
28 + torch.ops._C.silu_and_mul(out, x)
29
30
31 def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
20 - vllm_ops.gelu_and_mul(out, x)
32 + torch.ops._C.gelu_and_mul(out, x)
33
34
35 def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
24 - vllm_ops.gelu_tanh_and_mul(out, x)
36 + torch.ops._C.gelu_tanh_and_mul(out, x)
37
38
39 def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
28 - vllm_ops.gelu_fast(out, x)
40 + torch.ops._C.gelu_fast(out, x)
41
42
43 def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
32 - vllm_ops.gelu_new(out, x)
44 + torch.ops._C.gelu_new(out, x)
45
46
47 # page attention ops
@@ -53,7 +65,7 @@ def paged_attention_v1(
65 blocksparse_block_size: int = 64,
66 blocksparse_head_sliding_step: int = 0,
67 ) -> None:
56 - vllm_ops.paged_attention_v1(
68 + torch.ops._C.paged_attention_v1(
69 out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
70 seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
71 kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
@@ -83,7 +95,7 @@ def paged_attention_v2(
95 blocksparse_block_size: int = 64,
96 blocksparse_head_sliding_step: int = 0,
97 ) -> None:
86 - vllm_ops.paged_attention_v2(
98 + torch.ops._C.paged_attention_v2(
99 out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
100 num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
101 alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
@@ -100,8 +112,8 @@ def rotary_embedding(
112 cos_sin_cache: torch.Tensor,
113 is_neox: bool,
114 ) -> None:
103 - vllm_ops.rotary_embedding(positions, query, key, head_size, cos_sin_cache,
104 - is_neox)
115 + torch.ops._C.rotary_embedding(positions, query, key, head_size,
116 + cos_sin_cache, is_neox)
117
118
119 def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
@@ -109,20 +121,20 @@ def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
121 cos_sin_cache: torch.Tensor, is_neox: bool,
122 rot_dim: int,
123 cos_sin_cache_offsets: torch.Tensor) -> None:
112 - vllm_ops.batched_rotary_embedding(positions, query, key, head_size,
113 - cos_sin_cache, is_neox, rot_dim,
114 - cos_sin_cache_offsets)
124 + torch.ops._C.batched_rotary_embedding(positions, query, key, head_size,
125 + cos_sin_cache, is_neox, rot_dim,
126 + cos_sin_cache_offsets)
127
128
129 # layer norm ops
130 def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
131 epsilon: float) -> None:
120 - vllm_ops.rms_norm(out, input, weight, epsilon)
132 + torch.ops._C.rms_norm(out, input, weight, epsilon)
133
134
135 def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
136 weight: torch.Tensor, epsilon: float) -> None:
125 - vllm_ops.fused_add_rms_norm(input, residual, weight, epsilon)
137 + torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
138
139
140 # quantization ops
@@ -130,13 +142,13 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
142 def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
143 zeros: torch.Tensor, split_k_iters: int, thx: int,
144 thy: int) -> torch.Tensor:
133 - return vllm_ops.awq_dequantize(qweight, scales, zeros, split_k_iters, thx,
134 - thy)
145 + return torch.ops._C.awq_dequantize(qweight, scales, zeros, split_k_iters,
146 + thx, thy)
147
148
149 def awq_gemm(input: torch.Tensor, qweight: torch.Tensor, qzeros: torch.Tensor,
150 scales: torch.Tensor, split_k_iters: int) -> torch.Tensor:
139 - return vllm_ops.awq_gemm(input, qweight, qzeros, scales, split_k_iters)
151 + return torch.ops._C.awq_gemm(input, qweight, qzeros, scales, split_k_iters)
152
153
154 # gptq
@@ -144,27 +156,27 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
156 b_gptq_qzeros: torch.Tensor, b_gptq_scales: torch.Tensor,
157 b_g_idx: torch.Tensor, use_exllama: bool,
158 bit: int) -> torch.Tensor:
147 - return vllm_ops.gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
148 - b_g_idx, use_exllama, bit)
159 + return torch.ops._C.gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
160 + b_g_idx, use_exllama, bit)
161
162
163 def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
164 bit: int) -> None:
153 - vllm_ops.gptq_shuffle(q_weight, q_perm, bit)
165 + torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
166
167
168 # squeezellm
169 def squeezellm_gemm(vec: torch.Tensor, mat: torch.Tensor, mul: torch.Tensor,
170 lookup_table: torch.Tensor) -> None:
159 - vllm_ops.squeezellm_gemm(vec, mat, mul, lookup_table)
171 + torch.ops._C.squeezellm_gemm(vec, mat, mul, lookup_table)
172
173
174 # marlin
175 def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
176 b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int,
177 size_n: int, size_k: int) -> torch.Tensor:
166 - return vllm_ops.marlin_gemm(a, b_q_weight, b_scales, workspace, size_m,
167 - size_n, size_k)
178 + return torch.ops._C.marlin_gemm(a, b_q_weight, b_scales, workspace, size_m,
179 + size_n, size_k)
180
181
182 # marlin_24
@@ -172,9 +184,9 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
184 b_meta: torch.Tensor, b_scales: torch.Tensor,
185 workspace: torch.Tensor, num_bits: int, size_m: int,
186 size_n: int, size_k: int) -> torch.Tensor:
175 - return vllm_ops.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
176 - workspace, num_bits, size_m, size_n,
177 - size_k)
187 + return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
188 + workspace, num_bits, size_m,
189 + size_n, size_k)
190
191
192 # cutlass
@@ -188,7 +200,7 @@ def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
200 n = b.shape[1]
201 out = torch.empty((m, n), dtype=out_dtype, device=a.device)
202
191 - vllm_ops.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
203 + torch.ops._C.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
204
205 return out
206
@@ -198,21 +210,22 @@ def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
210 codebooks: torch.Tensor, scales: torch.Tensor,
211 codebook_partition_sizes: torch.Tensor,
212 bias: Optional[torch.Tensor]) -> torch.Tensor:
201 - return vllm_ops.aqlm_gemm(input, codes, codebooks, scales,
202 - codebook_partition_sizes, bias)
213 + return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,
214 + codebook_partition_sizes, bias)
215
216
217 def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
218 codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
207 - return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
219 + return torch.ops._C.aqlm_dequant(codes, codebooks,
220 + codebook_partition_sizes)
221
222
223 # gptq_marlin
224 def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
225 size_k: int, size_n: int,
226 num_bits: int) -> torch.Tensor:
214 - return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
215 - num_bits)
227 + return torch.ops._C.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
228 + num_bits)
229
230
231 def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -220,9 +233,9 @@ def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
233 perm: torch.Tensor, workspace: torch.Tensor,
234 num_bits: int, size_m: int, size_n: int, size_k: int,
235 is_k_full: bool) -> torch.Tensor:
223 - return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
224 - workspace, num_bits, size_m, size_n,
225 - size_k, is_k_full)
236 + return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
237 + workspace, num_bits, size_m, size_n,
238 + size_k, is_k_full)
239
240
241 # fp8
@@ -259,9 +272,9 @@ def scaled_fp8_quant(
272 output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
273 if scale is None:
274 scale = torch.zeros(1, device=input.device, dtype=torch.float32)
262 - vllm_ops.dynamic_scaled_fp8_quant(output, input, scale)
275 + torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
276 else:
264 - vllm_ops.static_scaled_fp8_quant(output, input, scale)
277 + torch.ops._C.static_scaled_fp8_quant(output, input, scale)
278 return output, scale
279
280
@@ -284,14 +297,14 @@ def scaled_int8_quant(
297 output = torch.empty_like(input, dtype=torch.int8)
298 if scale is not None:
299 # static-per-tensor quantization.
287 - vllm_ops.static_scaled_int8_quant(output, input, scale)
300 + torch.ops._C.static_scaled_int8_quant(output, input, scale)
301 return output, scale
302
303 # dynamic-per-token quantization.
304 input_scales = torch.empty((input.numel() // input.shape[-1], 1),
305 device=input.device,
306 dtype=torch.float32)
294 - vllm_ops.dynamic_scaled_int8_quant(output, input, input_scales)
307 + torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales)
308 return output, input_scales
309
310
@@ -300,9 +313,16 @@ def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
313 block_size: int, sorted_token_ids: torch.Tensor,
314 experts_ids: torch.Tensor,
315 num_tokens_post_pad: torch.Tensor) -> None:
303 - vllm_ops.moe_align_block_size(topk_ids, num_experts, block_size,
304 - sorted_token_ids, experts_ids,
305 - num_tokens_post_pad)
316 + torch.ops._C.moe_align_block_size(topk_ids, num_experts, block_size,
317 + sorted_token_ids, experts_ids,
318 + num_tokens_post_pad)
319 +
320 +
321 +def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
322 + token_expert_indicies: torch.Tensor,
323 + gating_output: float) -> None:
324 + torch.ops._moe_C.topk_softmax(topk_weights, topk_ids,
325 + token_expert_indicies, gating_output)
326
327
328 def reshape_and_cache(
@@ -314,8 +334,9 @@ def reshape_and_cache(
334 kv_cache_dtype: str,
335 kv_scale: float,
336 ) -> None:
317 - vllm_cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
318 - slot_mapping, kv_cache_dtype, kv_scale)
337 + torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
338 + value_cache, slot_mapping,
339 + kv_cache_dtype, kv_scale)
340
341
342 def reshape_and_cache_flash(
@@ -326,25 +347,115 @@ def reshape_and_cache_flash(
347 slot_mapping: torch.Tensor,
348 kv_cache_dtype: str,
349 ) -> None:
329 - vllm_cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
330 - slot_mapping, kv_cache_dtype)
350 + torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
351 + value_cache, slot_mapping,
352 + kv_cache_dtype)
353
354
355 def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
356 block_mapping: torch.Tensor) -> None:
335 - vllm_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
357 + torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
358
359
360 def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
361 block_mapping: torch.Tensor) -> None:
340 - vllm_cache_ops.swap_blocks(src, dst, block_mapping)
362 + torch.ops._C_cache_ops.swap_blocks(src, dst, block_mapping)
363
364
365 def convert_fp8(output: torch.Tensor,
366 input: torch.Tensor,
367 scale: float = 1.0,
368 kv_dtype: str = "fp8") -> None:
347 - vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
369 + torch.ops._C_cache_ops.convert_fp8(output, input, scale, kv_dtype)
370 +
371 +
372 +def get_device_attribute(attribute: int, device: int) -> int:
373 + return torch.ops._C_cuda_utils.get_device_attribute(attribute, device)
374 +
375 +
376 +def get_max_shared_memory_per_block_device_attribute(device: int) -> int:
377 + # ruff: noqa: E501
378 + return torch.ops._C_cuda_utils.get_max_shared_memory_per_block_device_attribute(
379 + device)
380 +
381 +
382 +# custom ar
383 +def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor,
384 + handles: List[str], offsets: List[int], rank: int,
385 + full_nvlink: bool) -> int:
386 + return torch.ops._C_custom_ar.init_custom_ar(meta, rank_data, handles,
387 + offsets, rank, full_nvlink)
388 +
389 +
390 +def should_custom_ar(inp: torch.Tensor, max_size: int, world_size: int,
391 + full_nvlink: bool) -> bool:
392 + return torch.ops._C_custom_ar.should_custom_ar(inp, max_size, world_size,
393 + full_nvlink)
394 +
395 +
396 +def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None:
397 + torch.ops._C_custom_ar.all_reduce_reg(fa, inp, out)
398 +
399
400 +def all_reduce_unreg(fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor,
401 + out: torch.Tensor) -> None:
402 + torch.ops._C_custom_ar.all_reduce_unreg(fa, inp, reg_buffer, out)
403
350 -#TODO: cuda_utils, custom_ar
404 +
405 +def dispose(fa: int) -> None:
406 + torch.ops._C_custom_ar.dispose(fa)
407 +
408 +
409 +def meta_size() -> int:
410 + return torch.ops._C_custom_ar.meta_size()
411 +
412 +
413 +def register_buffer(fa: int, t: torch.Tensor, handles: List[str],
414 + offsets: List[int]) -> None:
415 + return torch.ops._C_custom_ar.register_buffer(fa, t, handles, offsets)
416 +
417 +
418 +def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[str], List[int]]:
419 + return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa)
420 +
421 +
422 +def register_graph_buffers(fa: int, handles: List[str],
423 + offsets: List[List[int]]) -> None:
424 + torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
425 +
426 +
427 +# punica
428 +def dispatch_bgmv(
429 + y: torch.Tensor,
430 + x: torch.Tensor,
431 + w_t_all: torch.Tensor,
432 + indicies: torch.Tensor,
433 + layer_idx: int,
434 + scale: float,
435 +) -> None:
436 + torch.ops._punica_C.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx,
437 + scale)
438 +
439 +
440 +def dispatch_bgmv_low_level(
441 + y: torch.Tensor,
442 + x: torch.Tensor,
443 + w_t_all: torch.Tensor,
444 + indicies: torch.Tensor,
445 + layer_idx: int,
446 + scale: float,
447 + h_in: int,
448 + h_out: int,
449 + y_offset: int,
450 +) -> None:
451 + torch.ops._punica_C.dispatch_bgmv_low_level(
452 + y,
453 + x,
454 + w_t_all,
455 + indicies,
456 + layer_idx,
457 + scale,
458 + h_in,
459 + h_out,
460 + y_offset,
461 + )
vllm/attention/backends/flash_attn.py
+5 -5
@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
5 import torch
6 from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8 -from vllm._C import cache_ops
8 +from vllm import _custom_ops as ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata)
11
@@ -47,11 +47,11 @@ class FlashAttentionBackend(AttentionBackend):
47 ) -> None:
48 src_key_cache = src_kv_cache[0]
49 dst_key_cache = dst_kv_cache[0]
50 - cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
50 + ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
51
52 src_value_cache = src_kv_cache[1]
53 dst_value_cache = dst_kv_cache[1]
54 - cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
54 + ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
55
56 @staticmethod
57 def copy_blocks(
@@ -60,7 +60,7 @@ class FlashAttentionBackend(AttentionBackend):
60 ) -> None:
61 key_caches = [kv_cache[0] for kv_cache in kv_caches]
62 value_caches = [kv_cache[1] for kv_cache in kv_caches]
63 - cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
63 + ops.copy_blocks(key_caches, value_caches, src_to_dists)
64
65
66 @dataclass
@@ -285,7 +285,7 @@ class FlashAttentionImpl(AttentionImpl):
285 # Reshape the input keys and values and store them in the cache.
286 # If kv_cache is not provided, the new key and value tensors are
287 # not cached. This happens during the initial memory profiling run.
288 - cache_ops.reshape_and_cache_flash(
288 + ops.reshape_and_cache_flash(
289 key,
290 value,
291 key_cache,
vllm/distributed/device_communicators/custom_all_reduce.py
+19 -15
@@ -6,6 +6,7 @@ import torch.distributed as dist
6 from torch.distributed import ProcessGroup
7
8 import vllm.envs as envs
9 +from vllm import _custom_ops as ops
10 from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 from vllm.distributed.parallel_state import (
@@ -15,7 +16,11 @@ from vllm.logger import init_logger
16 try:
17 import pynvml
18
18 - from vllm._C import custom_ar
19 + # Simulate ImportError if custom_ar ops are not supported.
20 + if not ops.is_custom_op_supported("_C_custom_ar::meta_size"):
21 + raise ImportError("custom_ar", __file__)
22 +
23 + custom_ar = True
24
25 @contextmanager
26 def _nvml():
@@ -27,7 +32,7 @@ try:
32
33 except ImportError:
34 # For AMD GPUs
30 - custom_ar = None
35 + custom_ar = False
36 pynvml = None
37
38 @contextmanager
@@ -97,7 +102,7 @@ class CustomAllreduce:
102 self._IS_CAPTURING = False
103 self.disabled = True
104
100 - if custom_ar is None:
105 + if not custom_ar:
106 # disable because of missing custom allreduce library
107 # e.g. in a non-cuda environment
108 return
@@ -175,7 +180,7 @@ class CustomAllreduce:
180 # meta data composes of two parts: meta data for synchronization
181 # (256 bytes) and a temporary buffer for storing intermediate
182 # allreduce results.
178 - self.meta = torch.zeros(custom_ar.meta_size() + max_size,
183 + self.meta = torch.zeros(ops.meta_size() + max_size,
184 dtype=torch.uint8,
185 device=self.device)
186 # This is a pre-registered IPC buffer. In eager mode, input tensors
@@ -196,9 +201,8 @@ class CustomAllreduce:
201 self.world_size = world_size
202 handles, offsets = self._get_ipc_meta(self.meta)
203 self.full_nvlink = full_nvlink
199 - self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
200 - handles, offsets, rank,
201 - self.full_nvlink)
204 + self._ptr = ops.init_custom_ar(self.meta, self.rank_data, handles,
205 + offsets, rank, self.full_nvlink)
206 self.register_buffer(self.buffer)
207
208 @contextmanager
@@ -252,31 +256,31 @@ class CustomAllreduce:
256
257 def register_buffer(self, inp: torch.Tensor):
258 handles, offsets = self._get_ipc_meta(inp)
255 - custom_ar.register_buffer(self._ptr, inp, handles, offsets)
259 + ops.register_buffer(self._ptr, inp, handles, offsets)
260
261 def register_graph_buffers(self):
258 - handle, offset = custom_ar.get_graph_buffer_ipc_meta(self._ptr)
262 + handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
263 handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
264 logger.info("Registering %d cuda graph addresses", len(offset))
261 - custom_ar.register_graph_buffers(self._ptr, handles, offsets)
265 + ops.register_graph_buffers(self._ptr, handles, offsets)
266
267 def should_custom_ar(self, inp: torch.Tensor):
264 - return custom_ar.should_custom_ar(inp, self.max_size, self.world_size,
265 - self.full_nvlink)
268 + return ops.should_custom_ar(inp, self.max_size, self.world_size,
269 + self.full_nvlink)
270
271 # all reduce, assuming inp tensor is IPC registered with register_buffer,
272 # or, in the context of cuda graphs, register_graph_buffers
273 def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
274 if out is None:
275 out = torch.empty_like(inp)
272 - custom_ar.all_reduce_reg(self._ptr, inp, out)
276 + ops.all_reduce_reg(self._ptr, inp, out)
277 return out
278
279 # all reduce, assuming inp tensor is NOT IPC registered
280 def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
281 if out is None:
282 out = torch.empty_like(inp)
279 - custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
283 + ops.all_reduce_unreg(self._ptr, inp, self.buffer, out)
284 return out
285
286 def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
@@ -304,7 +308,7 @@ class CustomAllreduce:
308
309 def close(self):
310 if not self.disabled and self._ptr:
307 - custom_ar.dispose(self._ptr)
311 + ops.dispose(self._ptr)
312 self._ptr = 0
313
314 def __del__(self):
vllm/lora/punica.py
+19 -26
@@ -4,16 +4,21 @@ from typing import Optional
4
5 import torch
6
7 +from vllm import _custom_ops as ops
8 +
9 +
10 +def _check_punica_support():
11 + if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
12 + return
13
8 -def _raise_import_error(e):
14 if torch.cuda.get_device_capability() < (8, 0):
15 raise ImportError(
11 - "punica LoRA kernels require compute capability >= 8.0") from e
16 + "punica LoRA kernels require compute capability >= 8.0")
17 else:
18 raise ImportError(
19 "punica LoRA kernels could not be imported. If you built vLLM "
20 "from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
16 - "was set.") from e
21 + "was set.")
22
23
24 def bgmv(
@@ -41,12 +46,9 @@ def bgmv(
46 layer_idx: Layer index of the weight matrices.
47 scale: Scaling factor.
48 """
44 - try:
45 - import vllm._punica_C as punica_kernels
46 - except ImportError as e:
47 - _raise_import_error(e)
49 + _check_punica_support()
50
49 - punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
51 + ops.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
52
53
54 def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
@@ -75,11 +77,9 @@ def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
77 y_offset: Offset to apply to the starting column of y.
78 y_slice_size: Size of the y column slice.
79 """
78 - try:
79 - import vllm._punica_C as punica_kernels
80 - except ImportError as e:
81 - _raise_import_error(e)
82 - punica_kernels.dispatch_bgmv_low_level(
80 + _check_punica_support()
81 +
82 + ops.dispatch_bgmv_low_level(
83 y,
84 x,
85 w_t_all,
@@ -122,10 +122,7 @@ def add_lora(y: torch.Tensor,
122 scale: Scaling factor.
123 buffer: Optional. Shape: `[B, R]`. Temporary buffer.
124 """
125 - try:
126 - import vllm._punica_C as punica_kernels
127 - except ImportError as e:
128 - _raise_import_error(e)
125 + _check_punica_support()
126
127 r = wb_t_all.size(-1)
128 if buffer is None:
@@ -135,9 +132,8 @@ def add_lora(y: torch.Tensor,
132 buffer = torch.zeros((x.size(0), r),
133 dtype=torch.float32,
134 device=x.device)
138 - punica_kernels.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
139 - punica_kernels.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx,
140 - scale)
135 + ops.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
136 + ops.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx, scale)
137
138
139 def add_lora_slice(y: torch.Tensor,
@@ -176,10 +172,7 @@ def add_lora_slice(y: torch.Tensor,
172 y_offset: Offset to apply to the starting column of y.
173 y_slice_size: Size of the y column slice.
174 """
179 - try:
180 - import vllm._punica_C as punica_kernels
181 - except ImportError as e:
182 - _raise_import_error(e)
175 + _check_punica_support()
176
177 r = wb_t_all.size(-1)
178 if buffer is None:
@@ -189,7 +182,7 @@ def add_lora_slice(y: torch.Tensor,
182 buffer = torch.zeros((x.size(0), r),
183 dtype=torch.float32,
184 device=x.device)
192 - punica_kernels.dispatch_bgmv_low_level(
185 + ops.dispatch_bgmv_low_level(
186 buffer,
187 x,
188 wa_t_all,
@@ -200,7 +193,7 @@ def add_lora_slice(y: torch.Tensor,
193 buffer.size(1),
194 0,
195 )
203 - punica_kernels.dispatch_bgmv_low_level(
196 + ops.dispatch_bgmv_low_level(
197 y,
198 buffer,
199 wb_t_all,
vllm/model_executor/layers/fused_moe/fused_moe.py
+1 -2
@@ -8,7 +8,6 @@ import torch
8 import triton
9 import triton.language as tl
10
11 -import vllm._moe_C as moe_kernels
11 from vllm import _custom_ops as ops
12 from vllm.logger import init_logger
13
@@ -355,7 +354,7 @@ def fused_topk(
354 topk,
355 dtype=torch.int32,
356 device=hidden_states.device)
358 - moe_kernels.topk_softmax(
357 + ops.topk_softmax(
358 topk_weights,
359 topk_ids,
360 token_expert_indicies,
vllm/utils.py
+2 -5
@@ -22,6 +22,7 @@ import psutil
22 import torch
23
24 import vllm.envs as envs
25 +from vllm import _custom_ops as ops
26 from vllm.logger import enable_trace_function_call, init_logger
27
28 T = TypeVar("T")
@@ -148,12 +149,8 @@ def is_neuron() -> bool:
149 @lru_cache(maxsize=None)
150 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
151 """Returns the maximum shared memory per thread block in bytes."""
151 - # NOTE: This import statement should be executed lazily since
152 - # the Neuron-X backend does not have the `cuda_utils` module.
153 - from vllm._C import cuda_utils
154 -
152 max_shared_mem = (
156 - cuda_utils.get_max_shared_memory_per_block_device_attribute(gpu))
153 + ops.get_max_shared_memory_per_block_device_attribute(gpu))
154 # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py
155 # will fail
156 assert max_shared_mem > 0, "max_shared_mem can not be zero"