A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel][Misc] Use TORCH_LIBRARY instead of PYBIND11_MODULE for custom ops (#5047)
bnellnm committed
Jun 9, 2024 at 16:23 UTC
5467ac319636245ded483b31967ac43e543c5fa3
55 files changed
+833
-451
CMakeLists.txt
+6
-16
@@ -66,19 +66,6 @@ endif()
66
#
67
find_package(Torch REQUIRED)
68
69
-#
70
-# Normally `torch.utils.cpp_extension.CUDAExtension` would add
71
-# `libtorch_python.so` for linking against an extension. Torch's cmake
72
-# configuration does not include this library (presumably since the cmake
73
-# config is used for standalone C++ binaries that link against torch).
74
-# The `libtorch_python.so` library defines some of the glue code between
75
-# torch/python via pybind and is required by VLLM extensions for this
76
-# reason. So, add it by manually with `find_library` using torch's
77
-# installed library path.
78
-#
79
-find_library(torch_python_LIBRARY torch_python PATHS
80
- "${TORCH_INSTALL_PREFIX}/lib")
81
-
69
#
70
# Forward the non-CUDA device extensions to external CMake scripts.
71
#
@@ -171,7 +158,7 @@ set(VLLM_EXT_SRC
158
"csrc/quantization/fp8/common.cu"
159
"csrc/cuda_utils_kernels.cu"
160
"csrc/moe_align_block_size_kernels.cu"
174
- "csrc/pybind.cpp")
161
+ "csrc/torch_bindings.cpp")
162
163
if(VLLM_GPU_LANG STREQUAL "CUDA")
164
include(FetchContent)
@@ -218,6 +205,7 @@ define_gpu_extension_target(
205
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
206
ARCHITECTURES ${VLLM_GPU_ARCHES}
207
INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
208
+ USE_SABI 3
209
WITH_SOABI)
210
211
#
@@ -225,7 +213,7 @@ define_gpu_extension_target(
213
#
214
215
set(VLLM_MOE_EXT_SRC
228
- "csrc/moe/moe_ops.cpp"
216
+ "csrc/moe/torch_bindings.cpp"
217
"csrc/moe/topk_softmax_kernels.cu")
218
219
define_gpu_extension_target(
@@ -235,6 +223,7 @@ define_gpu_extension_target(
223
SOURCES ${VLLM_MOE_EXT_SRC}
224
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
225
ARCHITECTURES ${VLLM_GPU_ARCHES}
226
+ USE_SABI 3
227
WITH_SOABI)
228
229
#
@@ -249,7 +238,7 @@ set(VLLM_PUNICA_EXT_SRC
238
"csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
239
"csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
240
"csrc/punica/punica_ops.cu"
252
- "csrc/punica/punica_pybind.cpp")
241
+ "csrc/punica/torch_bindings.cpp")
242
243
#
244
# Copy GPU compilation flags+update for punica
@@ -286,6 +275,7 @@ if (VLLM_PUNICA_GPU_ARCHES)
275
SOURCES ${VLLM_PUNICA_EXT_SRC}
276
COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
277
ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
278
+ USE_SABI 3
279
WITH_SOABI)
280
else()
281
message(WARNING "Unable to create _punica_C target because none of the "
Dockerfile.rocm
+3
-3
@@ -106,9 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
106
pip install -U -r requirements-rocm.txt \
107
&& patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
108
&& python3 setup.py install \
109
- && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110
- && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111
- && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.cpython-39-x86_64-linux-gnu.so vllm/ \
109
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_C.abi3.so vllm/ \
110
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.abi3.so vllm/ \
111
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.abi3.so vllm/ \
112
&& cd ..
113
114
cmake/cpu_extension.cmake
+6
-6
@@ -12,7 +12,7 @@ include_directories("${CMAKE_SOURCE_DIR}/csrc")
12
#
13
# Check the compile flags
14
#
15
-list(APPEND CXX_COMPILE_FLAGS
15
+list(APPEND CXX_COMPILE_FLAGS
16
"-fopenmp"
17
"-DVLLM_CPU_EXTENSION")
18
@@ -44,8 +44,8 @@ if (AVX512_FOUND)
44
45
find_isa(${CPUINFO} "avx512_bf16" AVX512BF16_FOUND)
46
if (AVX512BF16_FOUND OR ENABLE_AVX512BF16)
47
- if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
48
- CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)
47
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
48
+ CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)
49
list(APPEND CXX_COMPILE_FLAGS "-mavx512bf16")
50
else()
51
message(WARNING "Disable AVX512-BF16 ISA support, requires gcc/g++ >= 12.3")
@@ -73,7 +73,7 @@ set(VLLM_EXT_SRC
73
"csrc/cpu/cache.cpp"
74
"csrc/cpu/layernorm.cpp"
75
"csrc/cpu/pos_encoding.cpp"
76
- "csrc/cpu/pybind.cpp")
76
+ "csrc/cpu/torch_bindings.cpp")
77
78
define_gpu_extension_target(
79
_C
@@ -81,10 +81,10 @@ define_gpu_extension_target(
81
LANGUAGE CXX
82
SOURCES ${VLLM_EXT_SRC}
83
COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
84
- WITH_SOABI
84
+ USE_SABI 3
85
+ WITH_SOABI
86
)
87
88
add_custom_target(default)
89
message(STATUS "Enabling C extension.")
90
add_dependencies(default _C)
90
-
cmake/utils.cmake
+8
-3
@@ -5,7 +5,7 @@
5
macro (find_python_from_executable EXECUTABLE SUPPORTED_VERSIONS)
6
file(REAL_PATH ${EXECUTABLE} EXECUTABLE)
7
set(Python_EXECUTABLE ${EXECUTABLE})
8
- find_package(Python COMPONENTS Interpreter Development.Module)
8
+ find_package(Python COMPONENTS Interpreter Development.Module Development.SABIModule)
9
if (NOT Python_FOUND)
10
message(FATAL_ERROR "Unable to find python matching: ${EXECUTABLE}.")
11
endif()
@@ -294,6 +294,7 @@ endmacro()
294
# INCLUDE_DIRECTORIES <dirs> - Extra include directories.
295
# LIBRARIES <libraries> - Extra link libraries.
296
# WITH_SOABI - Generate library with python SOABI suffix name.
297
+# USE_SABI <version> - Use python stable api <version>
298
#
299
# Note: optimization level/debug info is set via cmake build type.
300
#
@@ -301,7 +302,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
302
cmake_parse_arguments(PARSE_ARGV 1
303
GPU
304
"WITH_SOABI"
304
- "DESTINATION;LANGUAGE"
305
+ "DESTINATION;LANGUAGE;USE_SABI"
306
"SOURCES;ARCHITECTURES;COMPILE_FLAGS;INCLUDE_DIRECTORIES;LIBRARIES")
307
308
# Add hipify preprocessing step when building with HIP/ROCm.
@@ -315,7 +316,11 @@ function (define_gpu_extension_target GPU_MOD_NAME)
316
set(GPU_WITH_SOABI)
317
endif()
318
318
- Python_add_library(${GPU_MOD_NAME} MODULE "${GPU_SOURCES}" ${GPU_WITH_SOABI})
319
+ if (GPU_USE_SABI)
320
+ Python_add_library(${GPU_MOD_NAME} MODULE USE_SABI ${GPU_USE_SABI} ${GPU_WITH_SOABI} "${GPU_SOURCES}")
321
+ else()
322
+ Python_add_library(${GPU_MOD_NAME} MODULE ${GPU_WITH_SOABI} "${GPU_SOURCES}")
323
+ endif()
324
325
if (GPU_LANGUAGE STREQUAL "HIP")
326
# Make this target dependent on the hipify preprocessor step.
csrc/activation_kernels.cu
+1
-1
@@ -1,5 +1,5 @@
1
#include <ATen/cuda/CUDAContext.h>
2
-#include <torch/extension.h>
2
+#include <torch/all.h>
3
#include <c10/cuda/CUDAGuard.h>
4
5
#include <cmath>
csrc/attention/attention_kernels.cu
+18
-16
@@ -17,7 +17,7 @@
17
* limitations under the License.
18
*/
19
20
-#include <torch/extension.h>
20
+#include <torch/all.h>
21
#include <ATen/cuda/CUDAContext.h>
22
#include <c10/cuda/CUDAGuard.h>
23
#include <algorithm>
@@ -808,16 +808,17 @@ void paged_attention_v1(
808
torch::Tensor&
809
key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
810
torch::Tensor&
811
- value_cache, // [num_blocks, num_heads, head_size, block_size]
812
- int num_kv_heads, // [num_heads]
813
- float scale,
811
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
812
+ int64_t num_kv_heads, // [num_heads]
813
+ double scale,
814
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
815
torch::Tensor& seq_lens, // [num_seqs]
816
- int block_size, int max_seq_len,
816
+ int64_t block_size, int64_t max_seq_len,
817
const c10::optional<torch::Tensor>& alibi_slopes,
818
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
819
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
820
- const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
818
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
819
+ const int64_t blocksparse_local_blocks,
820
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
821
+ const int64_t blocksparse_head_sliding_step) {
822
const bool is_block_sparse = (blocksparse_vert_stride > 1);
823
824
DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
@@ -972,16 +973,17 @@ void paged_attention_v2(
973
torch::Tensor&
974
key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
975
torch::Tensor&
975
- value_cache, // [num_blocks, num_heads, head_size, block_size]
976
- int num_kv_heads, // [num_heads]
977
- float scale,
976
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
977
+ int64_t num_kv_heads, // [num_heads]
978
+ double scale,
979
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
980
torch::Tensor& seq_lens, // [num_seqs]
980
- int block_size, int max_seq_len,
981
+ int64_t block_size, int64_t max_seq_len,
982
const c10::optional<torch::Tensor>& alibi_slopes,
982
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
983
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
984
- const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
983
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
984
+ const int64_t blocksparse_local_blocks,
985
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
986
+ const int64_t blocksparse_head_sliding_step) {
987
const bool is_block_sparse = (blocksparse_vert_stride > 1);
988
DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
989
CALL_V2_LAUNCHER_BLOCK_SIZE)
@@ -990,4 +992,4 @@ void paged_attention_v2(
992
#undef WARP_SIZE
993
#undef MAX
994
#undef MIN
993
-#undef DIVIDE_ROUND_UP
\ No newline at end of file
995
+#undef DIVIDE_ROUND_UP
csrc/cache.h
+9
-5
@@ -1,6 +1,6 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+#include <torch/all.h>
4
5
#include <map>
6
#include <vector>
@@ -8,14 +8,18 @@
8
void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
9
const torch::Tensor& block_mapping);
10
11
-void copy_blocks(std::vector<torch::Tensor>& key_caches,
12
- std::vector<torch::Tensor>& value_caches,
11
+// Note: the key_caches and value_caches vectors are constant but
12
+// not the Tensors they contain. The vectors need to be const refs
13
+// in order to satisfy pytorch's C++ operator registration code.
14
+void copy_blocks(std::vector<torch::Tensor> const& key_caches,
15
+ std::vector<torch::Tensor> const& value_caches,
16
const torch::Tensor& block_mapping);
17
18
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19
torch::Tensor& key_cache, torch::Tensor& value_cache,
20
torch::Tensor& slot_mapping,
18
- const std::string& kv_cache_dtype, const float kv_scale);
21
+ const std::string& kv_cache_dtype,
22
+ const double kv_scale);
23
24
void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25
torch::Tensor& key_cache,
@@ -25,4 +29,4 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
29
30
// Just for unittest
31
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
28
- const float scale, const std::string& kv_cache_dtype);
32
+ const double scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+8
-5
@@ -1,4 +1,4 @@
1
-#include <torch/extension.h>
1
+#include <torch/all.h>
2
#include <ATen/cuda/CUDAContext.h>
3
#include <c10/cuda/CUDAGuard.h>
4
@@ -95,8 +95,11 @@ __global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
95
96
} // namespace vllm
97
98
-void copy_blocks(std::vector<torch::Tensor>& key_caches,
99
- std::vector<torch::Tensor>& value_caches,
98
+// Note: the key_caches and value_caches vectors are constant but
99
+// not the Tensors they contain. The vectors need to be const refs
100
+// in order to satisfy pytorch's C++ operator registration code.
101
+void copy_blocks(std::vector<torch::Tensor> const& key_caches,
102
+ std::vector<torch::Tensor> const& value_caches,
103
const torch::Tensor& block_mapping) {
104
int num_layers = key_caches.size();
105
TORCH_CHECK(num_layers == value_caches.size());
@@ -255,7 +258,7 @@ void reshape_and_cache(
258
torch::Tensor&
259
value_cache, // [num_blocks, num_heads, head_size, block_size]
260
torch::Tensor& slot_mapping, // [num_tokens]
258
- const std::string& kv_cache_dtype, const float kv_scale) {
261
+ const std::string& kv_cache_dtype, const double kv_scale) {
262
int num_tokens = key.size(0);
263
int num_heads = key.size(1);
264
int head_size = key.size(2);
@@ -334,7 +337,7 @@ __global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
337
338
// Only for testing.
339
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
337
- const float kv_scale, const std::string& kv_cache_dtype) {
340
+ const double kv_scale, const std::string& kv_cache_dtype) {
341
torch::Device src_device = src_cache.device();
342
torch::Device dst_device = dst_cache.device();
343
TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
csrc/cpu/attention.cpp
+14
-12
@@ -420,12 +420,13 @@ void paged_attention_v1_impl_launcher(
420
421
void paged_attention_v1(
422
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
423
- torch::Tensor& value_cache, int num_kv_heads, float scale,
424
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
425
- int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
427
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
428
- const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
423
+ torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
424
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
425
+ int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
427
+ const int64_t blocksparse_local_blocks,
428
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
429
+ const int64_t blocksparse_head_sliding_step) {
430
TORCH_CHECK(kv_scale == 1.0f);
431
TORCH_CHECK(blocksparse_vert_stride <= 1,
432
"CPU backend does not support blocksparse attention yet.");
@@ -738,12 +739,13 @@ void paged_attention_v2_impl_launcher(
739
void paged_attention_v2(
740
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
741
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
741
- torch::Tensor& value_cache, int num_kv_heads, float scale,
742
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
743
- int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
744
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
745
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
746
- const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
742
+ torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
743
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
744
+ int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
745
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
746
+ const int64_t blocksparse_local_blocks,
747
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
748
+ const int64_t blocksparse_head_sliding_step) {
749
TORCH_CHECK(kv_scale == 1.0f);
750
TORCH_CHECK(blocksparse_vert_stride <= 1,
751
"CPU backend does not support blocksparse attention yet.");
csrc/cpu/cache.cpp
+8
-5
@@ -5,8 +5,8 @@
5
6
namespace {
7
template <typename scalar_t>
8
-void copy_blocks_cpu_impl(std::vector<torch::Tensor>& key_caches,
9
- std::vector<torch::Tensor>& value_caches,
8
+void copy_blocks_cpu_impl(std::vector<torch::Tensor> const& key_caches,
9
+ std::vector<torch::Tensor> const& value_caches,
10
const torch::Tensor& mapping_pairs,
11
const int element_num_per_block,
12
const int layer_num) {
@@ -82,8 +82,11 @@ void reshape_and_cache_cpu_impl(
82
}
83
}; // namespace
84
85
-void copy_blocks(std::vector<torch::Tensor>& key_caches,
86
- std::vector<torch::Tensor>& value_caches,
85
+// Note: the key_caches and value_caches vectors are constant but
86
+// not the Tensors they contain. The vectors need to be const refs
87
+// in order to satisfy pytorch's C++ operator registration code.
88
+void copy_blocks(std::vector<torch::Tensor> const& key_caches,
89
+ std::vector<torch::Tensor> const& value_caches,
90
const torch::Tensor& block_mapping) {
91
unsigned num_layers = key_caches.size();
92
TORCH_CHECK(num_layers == value_caches.size());
@@ -104,7 +107,7 @@ void copy_blocks(std::vector<torch::Tensor>& key_caches,
107
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108
torch::Tensor& key_cache, torch::Tensor& value_cache,
109
torch::Tensor& slot_mapping,
107
- const std::string& kv_cache_dtype, float kv_scale) {
110
+ const std::string& kv_cache_dtype, double kv_scale) {
111
TORCH_CHECK(kv_scale == 1.0f);
112
113
int num_tokens = key.size(0);
csrc/cpu/cpu_types.hpp
+1
-1
@@ -3,7 +3,7 @@
3
#define CPU_TYPES_HPP
4
5
#include <immintrin.h>
6
-#include <torch/extension.h>
6
+#include <torch/all.h>
7
8
namespace vec_op {
9
csrc/cpu/layernorm.cpp
+2
-2
@@ -88,7 +88,7 @@ void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
88
} // namespace
89
90
void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
91
- float epsilon) {
91
+ double epsilon) {
92
int hidden_size = input.size(-1);
93
int num_tokens = input.numel() / hidden_size;
94
@@ -102,7 +102,7 @@ void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
102
}
103
104
void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
105
- torch::Tensor& weight, float epsilon) {
105
+ torch::Tensor& weight, double epsilon) {
106
int hidden_size = input.size(-1);
107
int num_tokens = input.numel() / hidden_size;
108
csrc/cpu/pos_encoding.cpp
+1
-1
@@ -168,7 +168,7 @@ void rotary_embedding_gptj_impl(
168
}; // namespace
169
170
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
171
- torch::Tensor& key, int head_size,
171
+ torch::Tensor& key, int64_t head_size,
172
torch::Tensor& cos_sin_cache, bool is_neox) {
173
int num_tokens = query.numel() / query.size(-1);
174
int rot_dim = cos_sin_cache.size(1);
csrc/cpu/pybind.cpp
deleted
-43
@@ -1,43 +0,0 @@
1
-#include "cache.h"
2
-#include "ops.h"
3
-#include <torch/extension.h>
4
-
5
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6
- // vLLM custom ops
7
- pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
8
-
9
- // Attention ops
10
- ops.def("paged_attention_v1", &paged_attention_v1,
11
- "Compute the attention between an input query and the cached "
12
- "keys/values using PagedAttention.");
13
- ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
14
-
15
- // Activation ops
16
- ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
17
- ops.def("gelu_and_mul", &gelu_and_mul,
18
- "Activation function used in GeGLU with `none` approximation.");
19
- ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
20
- "Activation function used in GeGLU with `tanh` approximation.");
21
- ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
22
- ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
23
-
24
- // Layernorm
25
- ops.def("rms_norm", &rms_norm,
26
- "Apply Root Mean Square (RMS) Normalization to the input tensor.");
27
-
28
- ops.def("fused_add_rms_norm", &fused_add_rms_norm,
29
- "In-place fused Add and RMS Normalization");
30
-
31
- // Rotary embedding
32
- ops.def("rotary_embedding", &rotary_embedding,
33
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
34
-
35
- // Cache ops
36
- pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
37
- cache_ops.def("swap_blocks", &swap_blocks,
38
- "Swap in (out) the cache blocks from src to dst");
39
- cache_ops.def("copy_blocks", ©_blocks,
40
- "Copy the cache blocks from src to dst");
41
- cache_ops.def("reshape_and_cache", &reshape_and_cache,
42
- "Reshape the key and value tensors and cache them");
43
-}
csrc/cpu/torch_bindings.cpp
new
+106
@@ -0,0 +1,106 @@
1
+#include "cache.h"
2
+#include "ops.h"
3
+#include "registration.h"
4
+
5
+#include <torch/library.h>
6
+
7
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
8
+ // vLLM custom ops
9
+
10
+ // Attention ops
11
+ // Compute the attention between an input query and the cached keys/values
12
+ // using PagedAttention.
13
+ ops.def(
14
+ "paged_attention_v1("
15
+ " Tensor! out, Tensor query, Tensor key_cache,"
16
+ " Tensor value_cache, int num_kv_heads, float scale,"
17
+ " Tensor block_tables, Tensor seq_lens, int block_size,"
18
+ " int max_seq_len, Tensor? alibi_slopes,"
19
+ " str kv_cache_dtype, float kv_scale, int tp_rank,"
20
+ " int blocksparse_local_blocks,"
21
+ " int blocksparse_vert_stride, int blocksparse_block_size,"
22
+ " int blocksparse_head_sliding_step) -> ()");
23
+ ops.impl("paged_attention_v1", torch::kCPU, &paged_attention_v1);
24
+
25
+ // PagedAttention V2.
26
+ ops.def(
27
+ "paged_attention_v2("
28
+ " Tensor! out, Tensor exp_sums, Tensor max_logits,"
29
+ " Tensor tmp_out, Tensor query, Tensor key_cache,"
30
+ " Tensor value_cache, int num_kv_heads, float scale,"
31
+ " Tensor block_tables, Tensor seq_lens, int block_size,"
32
+ " int max_seq_len, Tensor? alibi_slopes,"
33
+ " str kv_cache_dtype, float kv_scale, int tp_rank,"
34
+ " int blocksparse_local_blocks,"
35
+ " int blocksparse_vert_stride, int blocksparse_block_size,"
36
+ " int blocksparse_head_sliding_step) -> ()");
37
+ ops.impl("paged_attention_v2", torch::kCPU, &paged_attention_v2);
38
+
39
+ // Activation ops
40
+
41
+ // Activation function used in SwiGLU.
42
+ ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
43
+ ops.impl("silu_and_mul", torch::kCPU, &silu_and_mul);
44
+
45
+ // Activation function used in GeGLU with `none` approximation.
46
+ ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
47
+ ops.impl("gelu_and_mul", torch::kCPU, &gelu_and_mul);
48
+
49
+ // Activation function used in GeGLU with `tanh` approximation.
50
+ ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
51
+ ops.impl("gelu_tanh_and_mul", torch::kCPU, &gelu_tanh_and_mul);
52
+
53
+ // GELU implementation used in GPT-2.
54
+ ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
55
+ ops.impl("gelu_new", torch::kCPU, &gelu_new);
56
+
57
+ // Approximate GELU implementation.
58
+ ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
59
+ ops.impl("gelu_fast", torch::kCPU, &gelu_fast);
60
+
61
+ // Layernorm
62
+ // Apply Root Mean Square (RMS) Normalization to the input tensor.
63
+ ops.def(
64
+ "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
65
+ "()");
66
+ ops.impl("rms_norm", torch::kCPU, &rms_norm);
67
+
68
+ // In-place fused Add and RMS Normalization.
69
+ ops.def(
70
+ "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor weight, "
71
+ "float epsilon) -> ()");
72
+ ops.impl("fused_add_rms_norm", torch::kCPU, &fused_add_rms_norm);
73
+
74
+ // Rotary embedding
75
+ // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
76
+ ops.def(
77
+ "rotary_embedding(Tensor positions, Tensor! query,"
78
+ " Tensor! key, int head_size,"
79
+ " Tensor cos_sin_cache, bool is_neox) -> ()");
80
+ ops.impl("rotary_embedding", torch::kCPU, &rotary_embedding);
81
+}
82
+
83
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
84
+ // Cache ops
85
+ // Swap in (out) the cache blocks from src to dst.
86
+ cache_ops.def(
87
+ "swap_blocks(Tensor src, Tensor! dst, Tensor block_mapping) -> ()");
88
+ cache_ops.impl("swap_blocks", torch::kCPU, &swap_blocks);
89
+
90
+ // Copy the cache blocks from src to dst.
91
+ cache_ops.def(
92
+ "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
93
+ "block_mapping) -> ()");
94
+ cache_ops.impl("copy_blocks", torch::kCPU, ©_blocks);
95
+
96
+ // Reshape the key and value tensors and cache them.
97
+ cache_ops.def(
98
+ "reshape_and_cache(Tensor key, Tensor value,"
99
+ " Tensor! key_cache, Tensor! value_cache,"
100
+ " Tensor slot_mapping,"
101
+ " str kv_cache_dtype,"
102
+ " float kv_scale) -> ()");
103
+ cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
104
+}
105
+
106
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cuda_utils.h
+2
-4
@@ -1,7 +1,5 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+int64_t get_device_attribute(int64_t attribute, int64_t device_id);
4
5
-int get_device_attribute(int attribute, int device_id);
6
-
7
-int get_max_shared_memory_per_block_device_attribute(int device_id);
5
+int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id);
csrc/cuda_utils_kernels.cu
+3
-3
@@ -2,7 +2,7 @@
2
#include <hip/hip_runtime.h>
3
#include <hip/hip_runtime_api.h>
4
#endif
5
-int get_device_attribute(int attribute, int device_id) {
5
+int64_t get_device_attribute(int64_t attribute, int64_t device_id) {
6
int device, value;
7
if (device_id < 0) {
8
cudaGetDevice(&device);
@@ -14,8 +14,8 @@ int get_device_attribute(int attribute, int device_id) {
14
return value;
15
}
16
17
-int get_max_shared_memory_per_block_device_attribute(int device_id) {
18
- int attribute;
17
+int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id) {
18
+ int64_t attribute;
19
// https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
20
// cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
21
csrc/custom_all_reduce.cu
+14
-8
@@ -1,17 +1,17 @@
1
#include <ATen/cuda/Exceptions.h>
2
#include <c10/cuda/CUDAGuard.h>
3
#include <c10/cuda/CUDAStream.h>
4
-#include <torch/extension.h>
4
+#include <torch/all.h>
5
6
#include "custom_all_reduce.cuh"
7
8
-// fake pointer type
9
-using fptr_t = uint64_t;
8
+// fake pointer type, must match fptr_t type in ops.h
9
+using fptr_t = int64_t;
10
static_assert(sizeof(void*) == sizeof(fptr_t));
11
12
fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13
const std::vector<std::string>& handles,
14
- const std::vector<int64_t>& offsets, int rank,
14
+ const std::vector<int64_t>& offsets, int64_t rank,
15
bool full_nvlink) {
16
int world_size = offsets.size();
17
if (world_size > 8)
@@ -55,7 +55,7 @@ bool _is_weak_contiguous(torch::Tensor& t) {
55
t.numel() * t.element_size());
56
}
57
58
-bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
58
+bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
59
bool full_nvlink) {
60
auto inp_size = inp.numel() * inp.element_size();
61
// custom allreduce requires input byte size to be multiples of 16
@@ -125,7 +125,7 @@ void dispose(fptr_t _fa) {
125
delete fa;
126
}
127
128
-int meta_size() { return sizeof(vllm::Signal); }
128
+int64_t meta_size() { return sizeof(vllm::Signal); }
129
130
void register_buffer(fptr_t _fa, torch::Tensor& t,
131
const std::vector<std::string>& handles,
@@ -134,10 +134,16 @@ void register_buffer(fptr_t _fa, torch::Tensor& t,
134
fa->register_buffer(handles, offsets, t.data_ptr());
135
}
136
137
-std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
137
+std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
138
fptr_t _fa) {
139
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140
- return fa->get_graph_buffer_ipc_meta();
140
+ auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta();
141
+ auto options =
142
+ torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU);
143
+ auto handles =
144
+ torch::empty({static_cast<int64_t>(handle_bytes.size())}, options);
145
+ std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size());
146
+ return {handles, std::move(offsets)};
147
}
148
149
void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
csrc/dispatch_utils.h
+1
-1
@@ -4,7 +4,7 @@
4
*/
5
#pragma once
6
7
-#include <torch/extension.h>
7
+#include <torch/all.h>
8
9
#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10
AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
csrc/layernorm_kernels.cu
+3
-3
@@ -1,4 +1,4 @@
1
-#include <torch/extension.h>
1
+#include <torch/all.h>
2
#include <ATen/cuda/CUDAContext.h>
3
#include <c10/cuda/CUDAGuard.h>
4
@@ -291,7 +291,7 @@ fused_add_rms_norm_kernel(
291
void rms_norm(torch::Tensor& out, // [..., hidden_size]
292
torch::Tensor& input, // [..., hidden_size]
293
torch::Tensor& weight, // [hidden_size]
294
- float epsilon) {
294
+ double epsilon) {
295
int hidden_size = input.size(-1);
296
int num_tokens = input.numel() / hidden_size;
297
@@ -319,7 +319,7 @@ void rms_norm(torch::Tensor& out, // [..., hidden_size]
319
void fused_add_rms_norm(torch::Tensor& input, // [..., hidden_size]
320
torch::Tensor& residual, // [..., hidden_size]
321
torch::Tensor& weight, // [hidden_size]
322
- float epsilon) {
322
+ double epsilon) {
323
int hidden_size = input.size(-1);
324
int num_tokens = input.numel() / hidden_size;
325
csrc/moe/moe_ops.cpp
deleted
-8
@@ -1,8 +0,0 @@
1
-#include "moe_ops.h"
2
-
3
-#include <torch/extension.h>
4
-
5
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6
- m.def("topk_softmax", &topk_softmax,
7
- "Apply topk softmax to the gating outputs.");
8
-}
csrc/moe/moe_ops.h
+1
-1
@@ -1,6 +1,6 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+#include <torch/all.h>
4
5
void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6
torch::Tensor& token_expert_indices,
csrc/moe/topk_softmax_kernels.cu
+1
-1
@@ -16,7 +16,7 @@
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
18
*/
19
-#include <torch/extension.h>
19
+#include <torch/all.h>
20
#include <ATen/cuda/CUDAContext.h>
21
#include <c10/cuda/CUDAGuard.h>
22
#include "../cuda_compat.h"
csrc/moe/torch_bindings.cpp
new
+12
@@ -0,0 +1,12 @@
1
+#include "registration.h"
2
+#include "moe_ops.h"
3
+
4
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5
+ // Apply topk softmax to the gating outputs.
6
+ m.def(
7
+ "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
8
+ "token_expert_indices, Tensor gating_output) -> ()");
9
+ m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
10
+}
11
+
12
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/moe_align_block_size_kernels.cu
+3
-3
@@ -1,4 +1,4 @@
1
-#include <torch/extension.h>
1
+#include <torch/all.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
#include <ATen/ATen.h>
@@ -108,8 +108,8 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
108
}
109
} // namespace vllm
110
111
-void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
112
- int block_size, torch::Tensor sorted_token_ids,
111
+void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
112
+ int64_t block_size, torch::Tensor sorted_token_ids,
113
torch::Tensor experts_ids,
114
torch::Tensor num_tokens_post_pad) {
115
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
csrc/ops.h
+35
-33
@@ -1,40 +1,42 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+#include <torch/library.h>
4
5
void paged_attention_v1(
6
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
7
- torch::Tensor& value_cache, int num_kv_heads, float scale,
8
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
9
- int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
11
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
12
- const int blocksparse_block_size, const int blocksparse_head_sliding_step);
7
+ torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
8
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
9
+ int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
11
+ const int64_t blocksparse_local_blocks,
12
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
13
+ const int64_t blocksparse_head_sliding_step);
14
15
void paged_attention_v2(
16
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
17
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
17
- torch::Tensor& value_cache, int num_kv_heads, float scale,
18
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
19
- int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
20
- const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
21
- const int blocksparse_local_blocks, const int blocksparse_vert_stride,
22
- const int blocksparse_block_size, const int blocksparse_head_sliding_step);
18
+ torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
19
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
20
+ int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
21
+ const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
22
+ const int64_t blocksparse_local_blocks,
23
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
24
+ const int64_t blocksparse_head_sliding_step);
25
26
void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
25
- float epsilon);
27
+ double epsilon);
28
29
void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
28
- torch::Tensor& weight, float epsilon);
30
+ torch::Tensor& weight, double epsilon);
31
32
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
31
- torch::Tensor& key, int head_size,
33
+ torch::Tensor& key, int64_t head_size,
34
torch::Tensor& cos_sin_cache, bool is_neox);
35
36
void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
35
- torch::Tensor& key, int head_size,
37
+ torch::Tensor& key, int64_t head_size,
38
torch::Tensor& cos_sin_cache, bool is_neox,
37
- int rot_dim,
39
+ int64_t rot_dim,
40
torch::Tensor& cos_sin_cache_offsets);
41
42
void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
@@ -60,12 +62,12 @@ torch::Tensor aqlm_dequant(const torch::Tensor& codes,
62
63
torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
64
torch::Tensor _scaling_factors, torch::Tensor _zeros,
63
- int split_k_iters);
65
+ int64_t split_k_iters);
66
67
torch::Tensor awq_dequantize(torch::Tensor _kernel,
68
torch::Tensor _scaling_factors,
67
- torch::Tensor _zeros, int split_k_iters, int thx,
68
- int thy);
69
+ torch::Tensor _zeros, int64_t split_k_iters,
70
+ int64_t thx, int64_t thy);
71
72
torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
73
torch::Tensor& b_scales, torch::Tensor& workspace,
@@ -88,9 +90,9 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
90
int64_t size_k, int64_t size_n,
91
int64_t num_bits);
92
91
-int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
92
- torch::Tensor const& b, torch::Tensor const& a_scales,
93
- torch::Tensor const& b_scales);
93
+void cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
94
+ torch::Tensor const& b, torch::Tensor const& a_scales,
95
+ torch::Tensor const& b_scales);
96
97
#endif
98
@@ -106,9 +108,9 @@ void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
108
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
109
torch::Tensor b_gptq_qzeros,
110
torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
109
- bool use_exllama, int bit);
111
+ bool use_exllama, int64_t bit);
112
111
-void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit);
113
+void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
114
115
void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
116
torch::Tensor& scale);
@@ -116,28 +118,28 @@ void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
118
void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
119
torch::Tensor& scale);
120
119
-void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
120
- int block_size, torch::Tensor sorted_token_ids,
121
+void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
122
+ int64_t block_size, torch::Tensor sorted_token_ids,
123
torch::Tensor experts_ids,
124
torch::Tensor num_tokens_post_pad);
125
126
#ifndef USE_ROCM
125
-using fptr_t = uint64_t;
127
+using fptr_t = int64_t;
128
fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
129
const std::vector<std::string>& handles,
128
- const std::vector<int64_t>& offsets, int rank,
130
+ const std::vector<int64_t>& offsets, int64_t rank,
131
bool full_nvlink);
130
-bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
132
+bool should_custom_ar(torch::Tensor& inp, int64_t max_size, int64_t world_size,
133
bool full_nvlink);
134
void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
135
void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
136
torch::Tensor& out);
137
void dispose(fptr_t _fa);
136
-int meta_size();
138
+int64_t meta_size();
139
void register_buffer(fptr_t _fa, torch::Tensor& t,
140
const std::vector<std::string>& handles,
141
const std::vector<int64_t>& offsets);
140
-std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
142
+std::tuple<torch::Tensor, std::vector<int64_t>> get_graph_buffer_ipc_meta(
143
fptr_t _fa);
144
void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
145
const std::vector<std::vector<int64_t>>& offsets);
csrc/pos_encoding_kernels.cu
+6
-6
@@ -1,4 +1,4 @@
1
-#include <torch/extension.h>
1
+#include <torch/all.h>
2
#include <ATen/cuda/CUDAContext.h>
3
#include <c10/cuda/CUDAGuard.h>
4
@@ -127,7 +127,7 @@ void rotary_embedding(
127
// [num_tokens, num_heads * head_size]
128
torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
129
// [num_tokens, num_kv_heads * head_size]
130
- int head_size,
130
+ int64_t head_size,
131
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
132
bool is_neox) {
133
int64_t num_tokens = query.numel() / query.size(-1);
@@ -138,7 +138,7 @@ void rotary_embedding(
138
int64_t key_stride = key.stride(-2);
139
140
dim3 grid(num_tokens);
141
- dim3 block(std::min(num_heads * rot_dim / 2, 512));
141
+ dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
142
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
143
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
144
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
@@ -168,9 +168,9 @@ void batched_rotary_embedding(
168
// [num_tokens, num_heads * head_size]
169
torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
170
// [num_tokens, num_kv_heads * head_size]
171
- int head_size,
171
+ int64_t head_size,
172
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
173
- bool is_neox, int rot_dim,
173
+ bool is_neox, int64_t rot_dim,
174
torch::Tensor& cos_sin_cache_offsets // [num_tokens]
175
) {
176
int64_t num_tokens = cos_sin_cache_offsets.size(0);
@@ -180,7 +180,7 @@ void batched_rotary_embedding(
180
int64_t key_stride = key.stride(-2);
181
182
dim3 grid(num_tokens);
183
- dim3 block(std::min(num_heads * rot_dim / 2, 512));
183
+ dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
184
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
185
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
186
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
csrc/punica/punica_ops.cu
+3
-3
@@ -1,4 +1,4 @@
1
-#include <torch/extension.h>
1
+#include <torch/all.h>
2
#include <c10/cuda/CUDAGuard.h>
3
#include <cstdint>
4
@@ -88,7 +88,7 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
88
}
89
90
void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91
- torch::Tensor indicies, int64_t layer_idx, float scale) {
91
+ torch::Tensor indicies, int64_t layer_idx, double scale) {
92
CHECK_INPUT(y);
93
CHECK_INPUT(x);
94
CHECK_INPUT(w);
@@ -320,7 +320,7 @@ void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
320
321
void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
322
torch::Tensor indicies, int64_t layer_idx,
323
- float scale, int64_t h_in, int64_t h_out,
323
+ double scale, int64_t h_in, int64_t h_out,
324
int64_t y_offset) {
325
CHECK_INPUT(y);
326
CHECK_INPUT(x);
csrc/punica/punica_ops.h
+3
-3
@@ -1,11 +1,11 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+#include <torch/all.h>
4
5
void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6
- torch::Tensor indicies, int64_t layer_idx, float scale);
6
+ torch::Tensor indicies, int64_t layer_idx, double scale);
7
8
void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9
torch::Tensor indicies, int64_t layer_idx,
10
- float scale, int64_t h_in, int64_t h_out,
10
+ double scale, int64_t h_in, int64_t h_out,
11
int64_t y_offset);
csrc/punica/punica_pybind.cpp
deleted
-13
@@ -1,13 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-#include "punica_ops.h"
4
-
5
-//====== pybind ======
6
-
7
-#define DEFINE_pybind(name) m.def(#name, &name, #name);
8
-
9
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
10
- m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
11
- m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
12
- "dispatch_bgmv_low_level");
13
-}
csrc/punica/torch_bindings.cpp
new
+18
@@ -0,0 +1,18 @@
1
+#include "registration.h"
2
+#include "punica_ops.h"
3
+
4
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5
+ m.def(
6
+ "dispatch_bgmv(Tensor! y, Tensor x, Tensor w, Tensor indicies, int "
7
+ "layer_idx, float scale) -> ()");
8
+ m.impl("dispatch_bgmv", torch::kCUDA, &dispatch_bgmv);
9
+
10
+ m.def(
11
+ "dispatch_bgmv_low_level(Tensor! y, Tensor x, Tensor w,"
12
+ "Tensor indicies, int layer_idx,"
13
+ "float scale, int h_in, int h_out,"
14
+ "int y_offset) -> ()");
15
+ m.impl("dispatch_bgmv_low_level", torch::kCUDA, &dispatch_bgmv_low_level);
16
+}
17
+
18
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/pybind.cpp
deleted
-114
@@ -1,114 +0,0 @@
1
-#include "cache.h"
2
-#include "cuda_utils.h"
3
-#include "ops.h"
4
-#include <torch/extension.h>
5
-
6
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
7
- // vLLM custom ops
8
- pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
-
10
- // Attention ops
11
- ops.def("paged_attention_v1", &paged_attention_v1,
12
- "Compute the attention between an input query and the cached "
13
- "keys/values using PagedAttention.");
14
- ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15
-
16
- // Activation ops
17
- ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18
- ops.def("gelu_and_mul", &gelu_and_mul,
19
- "Activation function used in GeGLU with `none` approximation.");
20
- ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21
- "Activation function used in GeGLU with `tanh` approximation.");
22
- ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23
- ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24
-
25
- // Layernorm
26
- ops.def("rms_norm", &rms_norm,
27
- "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28
-
29
- ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30
- "In-place fused Add and RMS Normalization");
31
-
32
- // Rotary embedding
33
- ops.def("rotary_embedding", &rotary_embedding,
34
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35
-
36
- ops.def("batched_rotary_embedding", &batched_rotary_embedding,
37
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key "
38
- "(supports multiple loras)");
39
-
40
-// Quantization ops
41
-#ifndef USE_ROCM
42
- ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
43
- ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
44
- ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
45
- ops.def("marlin_gemm", &marlin_gemm,
46
- "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
47
- ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm,
48
- "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
49
- ops.def("gptq_marlin_gemm", &gptq_marlin_gemm,
50
- "gptq_marlin Optimized Quantized GEMM for GPTQ");
51
- ops.def("gptq_marlin_repack", &gptq_marlin_repack,
52
- "gptq_marlin repack from GPTQ");
53
- ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
54
- ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq,
55
- "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or "
56
- "per-row/column quantization.");
57
-#endif
58
-
59
- ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
60
- ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
61
- ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
62
- ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant,
63
- "Compute FP8 quantized tensor for given scaling factor");
64
- ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant,
65
- "Compute FP8 quantized tensor and scaling factor");
66
- ops.def("moe_align_block_size", &moe_align_block_size,
67
- "Aligning the number of tokens to be processed by each expert such "
68
- "that it is divisible by the block size.");
69
-
70
- ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71
- "Compute int8 quantized tensor for given scaling factor");
72
-
73
- ops.def("dynamic_scaled_int8_quant", &dynamic_scaled_int8_quant,
74
- "Compute int8 quantized tensor and scaling factor");
75
-
76
- // Cache ops
77
- pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
78
- cache_ops.def("swap_blocks", &swap_blocks,
79
- "Swap in (out) the cache blocks from src to dst");
80
- cache_ops.def("copy_blocks", ©_blocks,
81
- "Copy the cache blocks from src to dst");
82
- cache_ops.def("reshape_and_cache", &reshape_and_cache,
83
- "Reshape the key and value tensors and cache them");
84
- cache_ops.def("reshape_and_cache_flash", &reshape_and_cache_flash,
85
- "Reshape the key and value tensors and cache them");
86
- cache_ops.def("convert_fp8", &convert_fp8,
87
- "Convert the key and value cache to fp8 data type");
88
-
89
- // Cuda utils
90
- pybind11::module cuda_utils =
91
- m.def_submodule("cuda_utils", "vLLM cuda utils");
92
- cuda_utils.def("get_device_attribute", &get_device_attribute,
93
- "Gets the specified device attribute.");
94
-
95
- cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
96
- &get_max_shared_memory_per_block_device_attribute,
97
- "Gets the maximum shared memory per block device attribute.");
98
-
99
-#ifndef USE_ROCM
100
- // Custom all-reduce kernels
101
- pybind11::module custom_ar = m.def_submodule("custom_ar", "custom allreduce");
102
- custom_ar.def("init_custom_ar", &init_custom_ar, "init_custom_ar");
103
- custom_ar.def("should_custom_ar", &should_custom_ar, "should_custom_ar");
104
- custom_ar.def("all_reduce_reg", &all_reduce_reg, "all_reduce_reg");
105
- custom_ar.def("all_reduce_unreg", &all_reduce_unreg, "all_reduce_unreg");
106
- custom_ar.def("dispose", &dispose, "dispose");
107
- custom_ar.def("meta_size", &meta_size, "meta_size");
108
- custom_ar.def("register_buffer", ®ister_buffer, "register_buffer");
109
- custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta,
110
- "get_graph_buffer_ipc_meta");
111
- custom_ar.def("register_graph_buffers", ®ister_graph_buffers,
112
- "register_graph_buffers");
113
-#endif
114
-}
csrc/quantization/aqlm/gemm_kernels.cu
+1
-1
@@ -18,7 +18,7 @@
18
#include <cuda.h>
19
#include <cuda_fp16.h>
20
#include <cuda_runtime.h>
21
-#include <torch/extension.h>
21
+#include <torch/all.h>
22
#include <c10/cuda/CUDAStream.h>
23
#include <c10/cuda/CUDAGuard.h>
24
csrc/quantization/awq/gemm_kernels.cu
+4
-4
@@ -7,7 +7,7 @@ Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
7
}
8
*/
9
10
-#include <torch/extension.h>
10
+#include <torch/all.h>
11
#include <c10/cuda/CUDAGuard.h>
12
13
#include "dequantize.cuh"
@@ -435,8 +435,8 @@ __global__ void __launch_bounds__(64)
435
436
torch::Tensor awq_dequantize(torch::Tensor _kernel,
437
torch::Tensor _scaling_factors,
438
- torch::Tensor _zeros, int split_k_iters, int thx,
439
- int thy) {
438
+ torch::Tensor _zeros, int64_t split_k_iters,
439
+ int64_t thx, int64_t thy) {
440
int in_c = _kernel.size(0);
441
int qout_c = _kernel.size(1);
442
int out_c = qout_c * 8;
@@ -491,7 +491,7 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
491
492
torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
493
torch::Tensor _scaling_factors, torch::Tensor _zeros,
494
- int split_k_iters) {
494
+ int64_t split_k_iters) {
495
int num_in_feats = _in_feats.size(0);
496
int num_in_channels = _in_feats.size(1);
497
const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+1
-1
@@ -1,5 +1,5 @@
1
#include <ATen/cuda/CUDAContext.h>
2
-#include <torch/extension.h>
2
+#include <torch/all.h>
3
#include <cmath>
4
5
#include "../../dispatch_utils.h"
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+1
-1
@@ -1,5 +1,5 @@
1
#include <stddef.h>
2
-#include <torch/extension.h>
2
+#include <torch/all.h>
3
4
#include <ATen/cuda/CUDAContext.h>
5
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+1
-1
@@ -4,7 +4,7 @@
4
5
#if defined CUDA_VERSION && CUDA_VERSION >= 12000
6
7
-#include <torch/extension.h>
7
+#include <torch/all.h>
8
9
#include <ATen/cuda/CUDAContext.h>
10
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+1
-1
@@ -1,7 +1,7 @@
1
#include <cudaTypedefs.h>
2
3
#include <c10/cuda/CUDAGuard.h>
4
-#include <torch/extension.h>
4
+#include <torch/all.h>
5
6
void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
7
torch::Tensor const& b,
csrc/quantization/fp8/common.cu
+1
-1
@@ -1,5 +1,5 @@
1
#include <ATen/cuda/CUDAContext.h>
2
-#include <torch/extension.h>
2
+#include <torch/all.h>
3
#include <c10/cuda/CUDAGuard.h>
4
5
#include <cmath>
csrc/quantization/gptq/q_gemm.cu
+3
-3
@@ -6,7 +6,7 @@ https://github.com/qwopqwop200/GPTQ-for-LLaMa
6
#include <cstdint>
7
#include <cstdio>
8
9
-#include <torch/extension.h>
9
+#include <torch/all.h>
10
#include <c10/cuda/CUDAGuard.h>
11
#include <ATen/cuda/CUDAContext.h>
12
#include <cuda_runtime.h>
@@ -1823,7 +1823,7 @@ void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
1823
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1824
torch::Tensor b_gptq_qzeros,
1825
torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
1826
- bool use_exllama, int bit) {
1826
+ bool use_exllama, int64_t bit) {
1827
const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1828
auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1829
at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
@@ -1845,7 +1845,7 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1845
return c;
1846
}
1847
1848
-void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit) {
1848
+void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit) {
1849
const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
1850
vllm::gptq::shuffle_exllama_weight(
1851
(uint32_t*)q_weight.data_ptr(),
csrc/quantization/gptq_marlin/gptq_marlin.cu
+1
-1
@@ -1867,4 +1867,4 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1867
return c;
1868
}
1869
1870
-#endif
\ No newline at end of file
1870
+#endif
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+1
-1
@@ -1,6 +1,6 @@
1
#pragma once
2
3
-#include <torch/extension.h>
3
+#include <torch/all.h>
4
5
#include <ATen/cuda/CUDAContext.h>
6
#include <c10/cuda/CUDAGuard.h>
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+1
-1
@@ -15,7 +15,7 @@
15
* limitations under the License.
16
*/
17
18
-#include <torch/extension.h>
18
+#include <torch/all.h>
19
20
#include <ATen/cuda/CUDAContext.h>
21
#include <c10/cuda/CUDAGuard.h>
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+1
-1
@@ -16,7 +16,7 @@
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
18
*/
19
-#include <torch/extension.h>
19
+#include <torch/all.h>
20
21
#include <ATen/cuda/CUDAContext.h>
22
#include <c10/cuda/CUDAGuard.h>
csrc/quantization/squeezellm/quant_cuda_kernel.cu
-1
@@ -1,5 +1,4 @@
1
#include <torch/all.h>
2
-#include <torch/python.h>
2
#include <cuda.h>
3
#include <cuda_runtime.h>
4
#include <cuda_fp16.h>
csrc/registration.h
new
+22
@@ -0,0 +1,22 @@
1
+#pragma once
2
+
3
+#include <Python.h>
4
+
5
+#define _CONCAT(A, B) A##B
6
+#define CONCAT(A, B) _CONCAT(A, B)
7
+
8
+#define _STRINGIFY(A) #A
9
+#define STRINGIFY(A) _STRINGIFY(A)
10
+
11
+// A version of the TORCH_LIBRARY macro that expands the NAME, i.e. so NAME
12
+// could be a macro instead of a literal token.
13
+#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE)
14
+
15
+// REGISTER_EXTENSION allows the shared library to be loaded and initialized
16
+// via python's import statement.
17
+#define REGISTER_EXTENSION(NAME) \
18
+ PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \
19
+ static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, \
20
+ STRINGIFY(NAME), nullptr, 0, nullptr}; \
21
+ return PyModule_Create(&module); \
22
+ }
csrc/torch_bindings.cpp
new
+283
@@ -0,0 +1,283 @@
1
+#include "cache.h"
2
+#include "cuda_utils.h"
3
+#include "ops.h"
4
+#include "registration.h"
5
+
6
+#include <torch/library.h>
7
+
8
+// Note on op signatures:
9
+// The X_meta signatures are for the meta functions corresponding to op X.
10
+// They must be kept in sync with the signature for X. Generally, only
11
+// functions that return Tensors require a meta function.
12
+//
13
+// See the following links for detailed docs on op registration and function
14
+// schemas.
15
+// https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ptttacy8y1u9
16
+// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#annotations
17
+
18
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
19
+ // vLLM custom ops
20
+
21
+ // Attention ops
22
+ // Compute the attention between an input query and the cached
23
+ // keys/values using PagedAttention.
24
+ ops.def(
25
+ "paged_attention_v1("
26
+ " Tensor! out, Tensor query, Tensor key_cache,"
27
+ " Tensor value_cache, int num_kv_heads, float scale,"
28
+ " Tensor block_tables, Tensor seq_lens, int block_size,"
29
+ " int max_seq_len, Tensor? alibi_slopes,"
30
+ " str kv_cache_dtype, float kv_scale, int tp_rank,"
31
+ " int blocksparse_local_blocks,"
32
+ " int blocksparse_vert_stride, int blocksparse_block_size,"
33
+ " int blocksparse_head_sliding_step) -> ()");
34
+ ops.impl("paged_attention_v1", torch::kCUDA, &paged_attention_v1);
35
+
36
+ // PagedAttention V2.
37
+ ops.def(
38
+ "paged_attention_v2("
39
+ " Tensor! out, Tensor exp_sums, Tensor max_logits,"
40
+ " Tensor tmp_out, Tensor query, Tensor key_cache,"
41
+ " Tensor value_cache, int num_kv_heads, float scale,"
42
+ " Tensor block_tables, Tensor seq_lens, int block_size,"
43
+ " int max_seq_len, Tensor? alibi_slopes,"
44
+ " str kv_cache_dtype, float kv_scale, int tp_rank,"
45
+ " int blocksparse_local_blocks,"
46
+ " int blocksparse_vert_stride, int blocksparse_block_size,"
47
+ " int blocksparse_head_sliding_step) -> ()");
48
+ ops.impl("paged_attention_v2", torch::kCUDA, &paged_attention_v2);
49
+
50
+ // Activation ops
51
+ // Activation function used in SwiGLU.
52
+ ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
53
+ ops.impl("silu_and_mul", torch::kCUDA, &silu_and_mul);
54
+
55
+ // Activation function used in GeGLU with `none` approximation.
56
+ ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
57
+ ops.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul);
58
+
59
+ // Activation function used in GeGLU with `tanh` approximation.
60
+ ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
61
+ ops.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul);
62
+
63
+ // GELU implementation used in GPT-2.
64
+ ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
65
+ ops.impl("gelu_new", torch::kCUDA, &gelu_new);
66
+
67
+ // Approximate GELU implementation.
68
+ ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
69
+ ops.impl("gelu_fast", torch::kCUDA, &gelu_fast);
70
+
71
+ // Layernorm
72
+ // Apply Root Mean Square (RMS) Normalization to the input tensor.
73
+ ops.def(
74
+ "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
75
+ "()");
76
+ ops.impl("rms_norm", torch::kCUDA, &rms_norm);
77
+
78
+ // In-place fused Add and RMS Normalization.
79
+ ops.def(
80
+ "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor weight, "
81
+ "float epsilon) -> ()");
82
+ ops.impl("fused_add_rms_norm", torch::kCUDA, &fused_add_rms_norm);
83
+
84
+ // Rotary embedding
85
+ // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
86
+ ops.def(
87
+ "rotary_embedding(Tensor positions, Tensor! query,"
88
+ " Tensor! key, int head_size,"
89
+ " Tensor cos_sin_cache, bool is_neox) -> ()");
90
+ ops.impl("rotary_embedding", torch::kCUDA, &rotary_embedding);
91
+
92
+ // Apply GPT-NeoX or GPT-J style rotary embedding to query and key
93
+ // (supports multiple loras).
94
+ ops.def(
95
+ "batched_rotary_embedding(Tensor positions, Tensor! query,"
96
+ " Tensor! key, int head_size,"
97
+ " Tensor cos_sin_cache, bool is_neox,"
98
+ " int rot_dim,"
99
+ " Tensor cos_sin_cache_offsets) -> ()");
100
+ ops.impl("batched_rotary_embedding", torch::kCUDA, &batched_rotary_embedding);
101
+
102
+ // Quantization ops
103
+#ifndef USE_ROCM
104
+ // Quantized GEMM for AQLM.
105
+ ops.def("aqlm_gemm", &aqlm_gemm);
106
+ ops.impl("aqlm_gemm", torch::kCUDA, &aqlm_gemm);
107
+
108
+ // Decompression method for AQLM.
109
+ ops.def("aqlm_dequant", &aqlm_dequant);
110
+ ops.impl("aqlm_dequant", torch::kCUDA, &aqlm_dequant);
111
+
112
+ // Quantized GEMM for AWQ.
113
+ ops.def("awq_gemm", &awq_gemm);
114
+ ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
115
+
116
+ // Dequantization for AWQ.
117
+ ops.def("awq_dequantize", &awq_dequantize);
118
+ ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
119
+
120
+ // Marlin (Dense) Optimized Quantized GEMM for GPTQ.
121
+ ops.def("marlin_gemm", &marlin_gemm);
122
+ ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
123
+
124
+ // Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
125
+ ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
126
+ ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
127
+
128
+ // gptq_marlin Optimized Quantized GEMM for GPTQ.
129
+ ops.def("gptq_marlin_gemm", &gptq_marlin_gemm);
130
+ ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
131
+
132
+ // gptq_marlin repack from GPTQ.
133
+ ops.def("gptq_marlin_repack", &gptq_marlin_repack);
134
+ ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
135
+
136
+ // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
137
+ // quantization.
138
+ ops.def(
139
+ "cutlass_scaled_mm_dq(Tensor! out, Tensor a,"
140
+ " Tensor b, Tensor a_scales,"
141
+ " Tensor b_scales) -> ()");
142
+ ops.impl("cutlass_scaled_mm_dq", torch::kCUDA, &cutlass_scaled_mm_dq);
143
+#endif
144
+
145
+ // Quantized GEMM for GPTQ.
146
+ ops.def("gptq_gemm", &gptq_gemm);
147
+ ops.impl("gptq_gemm", torch::kCUDA, &gptq_gemm);
148
+
149
+ // Post processing for GPTQ.
150
+ ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
151
+ ops.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle);
152
+
153
+ // Quantized GEMM for SqueezeLLM.
154
+ ops.def(
155
+ "squeezellm_gemm(Tensor vec, Tensor mat, Tensor! mul, Tensor "
156
+ "lookup_table) -> ()");
157
+ ops.impl("squeezellm_gemm", torch::kCUDA, &squeezellm_gemm);
158
+
159
+ // Compute FP8 quantized tensor for given scaling factor.
160
+ ops.def(
161
+ "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
162
+ ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
163
+
164
+ // Compute FP8 quantized tensor and scaling factor.
165
+ ops.def(
166
+ "dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
167
+ "()");
168
+ ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
169
+
170
+ // Aligning the number of tokens to be processed by each expert such
171
+ // that it is divisible by the block size.
172
+ ops.def(
173
+ "moe_align_block_size(Tensor topk_ids, int num_experts,"
174
+ " int block_size, Tensor! sorted_token_ids,"
175
+ " Tensor! experts_ids,"
176
+ " Tensor! num_tokens_post_pad) -> ()");
177
+ ops.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size);
178
+
179
+ // Compute int8 quantized tensor for given scaling factor.
180
+ ops.def(
181
+ "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
182
+ "()");
183
+ ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
184
+
185
+ // Compute int8 quantized tensor and scaling factor
186
+ ops.def(
187
+ "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
188
+ "()");
189
+ ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
190
+ &dynamic_scaled_int8_quant);
191
+}
192
+
193
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
194
+ // Cache ops
195
+ // Swap in (out) the cache blocks from src to dst.
196
+ cache_ops.def(
197
+ "swap_blocks(Tensor src, Tensor! dst, Tensor block_mapping) -> ()");
198
+ cache_ops.impl("swap_blocks", torch::kCUDA, &swap_blocks);
199
+
200
+ // Copy the cache blocks from src to dst.
201
+ cache_ops.def(
202
+ "copy_blocks(Tensor[]! key_caches, Tensor[]! value_caches, Tensor "
203
+ "block_mapping) -> ()");
204
+ cache_ops.impl("copy_blocks", torch::kCUDA, ©_blocks);
205
+
206
+ // Reshape the key and value tensors and cache them.
207
+ cache_ops.def(
208
+ "reshape_and_cache(Tensor key, Tensor value,"
209
+ " Tensor! key_cache, Tensor! value_cache,"
210
+ " Tensor slot_mapping,"
211
+ " str kv_cache_dtype,"
212
+ " float kv_scale) -> ()");
213
+ cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
214
+
215
+ // Reshape the key and value tensors and cache them.
216
+ cache_ops.def(
217
+ "reshape_and_cache_flash(Tensor key, Tensor value,"
218
+ " Tensor! key_cache,"
219
+ " Tensor! value_cache,"
220
+ " Tensor slot_mapping,"
221
+ " str kv_cache_dtype) -> ()");
222
+ cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
223
+ &reshape_and_cache_flash);
224
+
225
+ // Convert the key and value cache to fp8 data type.
226
+ cache_ops.def(
227
+ "convert_fp8(Tensor! dst_cache, Tensor src_cache, float scale, str "
228
+ "kv_cache_dtype) -> ()");
229
+ cache_ops.impl("convert_fp8", torch::kCUDA, &convert_fp8);
230
+}
231
+
232
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
233
+ // Cuda utils
234
+
235
+ // Gets the specified device attribute.
236
+ cuda_utils.def("get_device_attribute", &get_device_attribute);
237
+ cuda_utils.impl("get_device_attribute", torch::kCUDA, &get_device_attribute);
238
+
239
+ // Gets the maximum shared memory per block device attribute.
240
+ cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
241
+ &get_max_shared_memory_per_block_device_attribute);
242
+ cuda_utils.impl("get_max_shared_memory_per_block_device_attribute",
243
+ torch::kCUDA,
244
+ &get_max_shared_memory_per_block_device_attribute);
245
+}
246
+
247
+#ifndef USE_ROCM
248
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _custom_ar), custom_ar) {
249
+ // Custom all-reduce kernels
250
+ custom_ar.def("init_custom_ar", &init_custom_ar);
251
+ custom_ar.impl("init_custom_ar", torch::kCUDA, &init_custom_ar);
252
+
253
+ custom_ar.def("should_custom_ar", &should_custom_ar);
254
+ custom_ar.impl("should_custom_ar", torch::kCUDA, &should_custom_ar);
255
+
256
+ custom_ar.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()");
257
+ custom_ar.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg);
258
+
259
+ custom_ar.def(
260
+ "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> "
261
+ "()");
262
+ custom_ar.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg);
263
+
264
+ custom_ar.def("dispose", &dispose);
265
+ custom_ar.impl("dispose", torch::kCPU, &dispose);
266
+
267
+ custom_ar.def("meta_size", &meta_size);
268
+ custom_ar.impl("meta_size", torch::kCPU, &meta_size);
269
+
270
+ custom_ar.def("register_buffer", ®ister_buffer);
271
+ custom_ar.impl("register_buffer", torch::kCUDA, ®ister_buffer);
272
+
273
+ custom_ar.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta);
274
+ custom_ar.impl("get_graph_buffer_ipc_meta", torch::kCPU,
275
+ &get_graph_buffer_ipc_meta);
276
+
277
+ custom_ar.def("register_graph_buffers", ®ister_graph_buffers);
278
+ custom_ar.impl("register_graph_buffers", torch::kCPU,
279
+ ®ister_graph_buffers);
280
+}
281
+#endif
282
+
283
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
setup.py
+1
-1
@@ -60,7 +60,7 @@ def remove_prefix(text, prefix):
60
class CMakeExtension(Extension):
61
62
def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
63
- super().__init__(name, sources=[], **kwa)
63
+ super().__init__(name, sources=[], py_limited_api=True, **kwa)
64
self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
65
66
tests/kernels/test_int8_quant.py
+4
-3
@@ -1,7 +1,8 @@
1
import pytest
2
import torch
3
4
-from vllm._C import ops
4
+# ruff: noqa: F401
5
+import vllm._C
6
7
DTYPES = [torch.half, torch.bfloat16, torch.float]
8
HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -33,7 +34,7 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
34
35
ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
36
scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
36
- ops.dynamic_scaled_int8_quant(ops_out, x, scales_out)
37
+ torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out)
38
39
assert torch.allclose(scales_out, scales)
40
assert torch.allclose(torch_out, ops_out,
@@ -60,6 +61,6 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
61
out2 = torch.empty_like(x, dtype=torch.int8)
62
scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
63
63
- ops.static_scaled_int8_quant(out2, x, scale_argument)
64
+ torch.ops._C.static_scaled_int8_quant(out2, x, scale_argument)
65
assert torch.allclose(out1, out2,
66
atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+164
-53
@@ -1,35 +1,47 @@
1
-from typing import Optional, Tuple, Type
1
+import contextlib
2
+from typing import List, Optional, Tuple, Type
3
4
import torch
5
6
try:
6
- from vllm._C import cache_ops as vllm_cache_ops
7
- from vllm._C import ops as vllm_ops
7
+ import vllm._C
8
except ImportError as e:
9
from vllm.logger import init_logger
10
logger = init_logger(__name__)
11
logger.warning("Failed to import from vllm._C with %r", e)
12
13
+with contextlib.suppress(ImportError):
14
+ import vllm._moe_C
15
+
16
+with contextlib.suppress(ImportError):
17
+ # ruff: noqa: F401
18
+ import vllm._punica_C
19
+
20
+
21
+def is_custom_op_supported(op_name: str) -> bool:
22
+ op, overloads = torch._C._jit_get_operation(op_name)
23
+ return op is not None
24
+
25
26
# activation ops
27
def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
16
- vllm_ops.silu_and_mul(out, x)
28
+ torch.ops._C.silu_and_mul(out, x)
29
30
31
def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
20
- vllm_ops.gelu_and_mul(out, x)
32
+ torch.ops._C.gelu_and_mul(out, x)
33
34
35
def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
24
- vllm_ops.gelu_tanh_and_mul(out, x)
36
+ torch.ops._C.gelu_tanh_and_mul(out, x)
37
38
39
def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
28
- vllm_ops.gelu_fast(out, x)
40
+ torch.ops._C.gelu_fast(out, x)
41
42
43
def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
32
- vllm_ops.gelu_new(out, x)
44
+ torch.ops._C.gelu_new(out, x)
45
46
47
# page attention ops
@@ -53,7 +65,7 @@ def paged_attention_v1(
65
blocksparse_block_size: int = 64,
66
blocksparse_head_sliding_step: int = 0,
67
) -> None:
56
- vllm_ops.paged_attention_v1(
68
+ torch.ops._C.paged_attention_v1(
69
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
70
seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
71
kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
@@ -83,7 +95,7 @@ def paged_attention_v2(
95
blocksparse_block_size: int = 64,
96
blocksparse_head_sliding_step: int = 0,
97
) -> None:
86
- vllm_ops.paged_attention_v2(
98
+ torch.ops._C.paged_attention_v2(
99
out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
100
num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
101
alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
@@ -100,8 +112,8 @@ def rotary_embedding(
112
cos_sin_cache: torch.Tensor,
113
is_neox: bool,
114
) -> None:
103
- vllm_ops.rotary_embedding(positions, query, key, head_size, cos_sin_cache,
104
- is_neox)
115
+ torch.ops._C.rotary_embedding(positions, query, key, head_size,
116
+ cos_sin_cache, is_neox)
117
118
119
def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
@@ -109,20 +121,20 @@ def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
121
cos_sin_cache: torch.Tensor, is_neox: bool,
122
rot_dim: int,
123
cos_sin_cache_offsets: torch.Tensor) -> None:
112
- vllm_ops.batched_rotary_embedding(positions, query, key, head_size,
113
- cos_sin_cache, is_neox, rot_dim,
114
- cos_sin_cache_offsets)
124
+ torch.ops._C.batched_rotary_embedding(positions, query, key, head_size,
125
+ cos_sin_cache, is_neox, rot_dim,
126
+ cos_sin_cache_offsets)
127
128
129
# layer norm ops
130
def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
131
epsilon: float) -> None:
120
- vllm_ops.rms_norm(out, input, weight, epsilon)
132
+ torch.ops._C.rms_norm(out, input, weight, epsilon)
133
134
135
def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
136
weight: torch.Tensor, epsilon: float) -> None:
125
- vllm_ops.fused_add_rms_norm(input, residual, weight, epsilon)
137
+ torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
138
139
140
# quantization ops
@@ -130,13 +142,13 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
142
def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
143
zeros: torch.Tensor, split_k_iters: int, thx: int,
144
thy: int) -> torch.Tensor:
133
- return vllm_ops.awq_dequantize(qweight, scales, zeros, split_k_iters, thx,
134
- thy)
145
+ return torch.ops._C.awq_dequantize(qweight, scales, zeros, split_k_iters,
146
+ thx, thy)
147
148
149
def awq_gemm(input: torch.Tensor, qweight: torch.Tensor, qzeros: torch.Tensor,
150
scales: torch.Tensor, split_k_iters: int) -> torch.Tensor:
139
- return vllm_ops.awq_gemm(input, qweight, qzeros, scales, split_k_iters)
151
+ return torch.ops._C.awq_gemm(input, qweight, qzeros, scales, split_k_iters)
152
153
154
# gptq
@@ -144,27 +156,27 @@ def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
156
b_gptq_qzeros: torch.Tensor, b_gptq_scales: torch.Tensor,
157
b_g_idx: torch.Tensor, use_exllama: bool,
158
bit: int) -> torch.Tensor:
147
- return vllm_ops.gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
148
- b_g_idx, use_exllama, bit)
159
+ return torch.ops._C.gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
160
+ b_g_idx, use_exllama, bit)
161
162
163
def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
164
bit: int) -> None:
153
- vllm_ops.gptq_shuffle(q_weight, q_perm, bit)
165
+ torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
166
167
168
# squeezellm
169
def squeezellm_gemm(vec: torch.Tensor, mat: torch.Tensor, mul: torch.Tensor,
170
lookup_table: torch.Tensor) -> None:
159
- vllm_ops.squeezellm_gemm(vec, mat, mul, lookup_table)
171
+ torch.ops._C.squeezellm_gemm(vec, mat, mul, lookup_table)
172
173
174
# marlin
175
def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
176
b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int,
177
size_n: int, size_k: int) -> torch.Tensor:
166
- return vllm_ops.marlin_gemm(a, b_q_weight, b_scales, workspace, size_m,
167
- size_n, size_k)
178
+ return torch.ops._C.marlin_gemm(a, b_q_weight, b_scales, workspace, size_m,
179
+ size_n, size_k)
180
181
182
# marlin_24
@@ -172,9 +184,9 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
184
b_meta: torch.Tensor, b_scales: torch.Tensor,
185
workspace: torch.Tensor, num_bits: int, size_m: int,
186
size_n: int, size_k: int) -> torch.Tensor:
175
- return vllm_ops.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
176
- workspace, num_bits, size_m, size_n,
177
- size_k)
187
+ return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
188
+ workspace, num_bits, size_m,
189
+ size_n, size_k)
190
191
192
# cutlass
@@ -188,7 +200,7 @@ def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
200
n = b.shape[1]
201
out = torch.empty((m, n), dtype=out_dtype, device=a.device)
202
191
- vllm_ops.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
203
+ torch.ops._C.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
204
205
return out
206
@@ -198,21 +210,22 @@ def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
210
codebooks: torch.Tensor, scales: torch.Tensor,
211
codebook_partition_sizes: torch.Tensor,
212
bias: Optional[torch.Tensor]) -> torch.Tensor:
201
- return vllm_ops.aqlm_gemm(input, codes, codebooks, scales,
202
- codebook_partition_sizes, bias)
213
+ return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,
214
+ codebook_partition_sizes, bias)
215
216
217
def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
218
codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
207
- return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
219
+ return torch.ops._C.aqlm_dequant(codes, codebooks,
220
+ codebook_partition_sizes)
221
222
223
# gptq_marlin
224
def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
225
size_k: int, size_n: int,
226
num_bits: int) -> torch.Tensor:
214
- return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
215
- num_bits)
227
+ return torch.ops._C.gptq_marlin_repack(b_q_weight, perm, size_k, size_n,
228
+ num_bits)
229
230
231
def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
@@ -220,9 +233,9 @@ def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
233
perm: torch.Tensor, workspace: torch.Tensor,
234
num_bits: int, size_m: int, size_n: int, size_k: int,
235
is_k_full: bool) -> torch.Tensor:
223
- return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
224
- workspace, num_bits, size_m, size_n,
225
- size_k, is_k_full)
236
+ return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
237
+ workspace, num_bits, size_m, size_n,
238
+ size_k, is_k_full)
239
240
241
# fp8
@@ -259,9 +272,9 @@ def scaled_fp8_quant(
272
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
273
if scale is None:
274
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
262
- vllm_ops.dynamic_scaled_fp8_quant(output, input, scale)
275
+ torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
276
else:
264
- vllm_ops.static_scaled_fp8_quant(output, input, scale)
277
+ torch.ops._C.static_scaled_fp8_quant(output, input, scale)
278
return output, scale
279
280
@@ -284,14 +297,14 @@ def scaled_int8_quant(
297
output = torch.empty_like(input, dtype=torch.int8)
298
if scale is not None:
299
# static-per-tensor quantization.
287
- vllm_ops.static_scaled_int8_quant(output, input, scale)
300
+ torch.ops._C.static_scaled_int8_quant(output, input, scale)
301
return output, scale
302
303
# dynamic-per-token quantization.
304
input_scales = torch.empty((input.numel() // input.shape[-1], 1),
305
device=input.device,
306
dtype=torch.float32)
294
- vllm_ops.dynamic_scaled_int8_quant(output, input, input_scales)
307
+ torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales)
308
return output, input_scales
309
310
@@ -300,9 +313,16 @@ def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
313
block_size: int, sorted_token_ids: torch.Tensor,
314
experts_ids: torch.Tensor,
315
num_tokens_post_pad: torch.Tensor) -> None:
303
- vllm_ops.moe_align_block_size(topk_ids, num_experts, block_size,
304
- sorted_token_ids, experts_ids,
305
- num_tokens_post_pad)
316
+ torch.ops._C.moe_align_block_size(topk_ids, num_experts, block_size,
317
+ sorted_token_ids, experts_ids,
318
+ num_tokens_post_pad)
319
+
320
+
321
+def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
322
+ token_expert_indicies: torch.Tensor,
323
+ gating_output: float) -> None:
324
+ torch.ops._moe_C.topk_softmax(topk_weights, topk_ids,
325
+ token_expert_indicies, gating_output)
326
327
328
def reshape_and_cache(
@@ -314,8 +334,9 @@ def reshape_and_cache(
334
kv_cache_dtype: str,
335
kv_scale: float,
336
) -> None:
317
- vllm_cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
318
- slot_mapping, kv_cache_dtype, kv_scale)
337
+ torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
338
+ value_cache, slot_mapping,
339
+ kv_cache_dtype, kv_scale)
340
341
342
def reshape_and_cache_flash(
@@ -326,25 +347,115 @@ def reshape_and_cache_flash(
347
slot_mapping: torch.Tensor,
348
kv_cache_dtype: str,
349
) -> None:
329
- vllm_cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
330
- slot_mapping, kv_cache_dtype)
350
+ torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
351
+ value_cache, slot_mapping,
352
+ kv_cache_dtype)
353
354
355
def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
356
block_mapping: torch.Tensor) -> None:
335
- vllm_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
357
+ torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
358
359
360
def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
361
block_mapping: torch.Tensor) -> None:
340
- vllm_cache_ops.swap_blocks(src, dst, block_mapping)
362
+ torch.ops._C_cache_ops.swap_blocks(src, dst, block_mapping)
363
364
365
def convert_fp8(output: torch.Tensor,
366
input: torch.Tensor,
367
scale: float = 1.0,
368
kv_dtype: str = "fp8") -> None:
347
- vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
369
+ torch.ops._C_cache_ops.convert_fp8(output, input, scale, kv_dtype)
370
+
371
+
372
+def get_device_attribute(attribute: int, device: int) -> int:
373
+ return torch.ops._C_cuda_utils.get_device_attribute(attribute, device)
374
+
375
+
376
+def get_max_shared_memory_per_block_device_attribute(device: int) -> int:
377
+ # ruff: noqa: E501
378
+ return torch.ops._C_cuda_utils.get_max_shared_memory_per_block_device_attribute(
379
+ device)
380
+
381
+
382
+# custom ar
383
+def init_custom_ar(meta: torch.Tensor, rank_data: torch.Tensor,
384
+ handles: List[str], offsets: List[int], rank: int,
385
+ full_nvlink: bool) -> int:
386
+ return torch.ops._C_custom_ar.init_custom_ar(meta, rank_data, handles,
387
+ offsets, rank, full_nvlink)
388
+
389
+
390
+def should_custom_ar(inp: torch.Tensor, max_size: int, world_size: int,
391
+ full_nvlink: bool) -> bool:
392
+ return torch.ops._C_custom_ar.should_custom_ar(inp, max_size, world_size,
393
+ full_nvlink)
394
+
395
+
396
+def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None:
397
+ torch.ops._C_custom_ar.all_reduce_reg(fa, inp, out)
398
+
399
400
+def all_reduce_unreg(fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor,
401
+ out: torch.Tensor) -> None:
402
+ torch.ops._C_custom_ar.all_reduce_unreg(fa, inp, reg_buffer, out)
403
350
-#TODO: cuda_utils, custom_ar
404
+
405
+def dispose(fa: int) -> None:
406
+ torch.ops._C_custom_ar.dispose(fa)
407
+
408
+
409
+def meta_size() -> int:
410
+ return torch.ops._C_custom_ar.meta_size()
411
+
412
+
413
+def register_buffer(fa: int, t: torch.Tensor, handles: List[str],
414
+ offsets: List[int]) -> None:
415
+ return torch.ops._C_custom_ar.register_buffer(fa, t, handles, offsets)
416
+
417
+
418
+def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[str], List[int]]:
419
+ return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa)
420
+
421
+
422
+def register_graph_buffers(fa: int, handles: List[str],
423
+ offsets: List[List[int]]) -> None:
424
+ torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
425
+
426
+
427
+# punica
428
+def dispatch_bgmv(
429
+ y: torch.Tensor,
430
+ x: torch.Tensor,
431
+ w_t_all: torch.Tensor,
432
+ indicies: torch.Tensor,
433
+ layer_idx: int,
434
+ scale: float,
435
+) -> None:
436
+ torch.ops._punica_C.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx,
437
+ scale)
438
+
439
+
440
+def dispatch_bgmv_low_level(
441
+ y: torch.Tensor,
442
+ x: torch.Tensor,
443
+ w_t_all: torch.Tensor,
444
+ indicies: torch.Tensor,
445
+ layer_idx: int,
446
+ scale: float,
447
+ h_in: int,
448
+ h_out: int,
449
+ y_offset: int,
450
+) -> None:
451
+ torch.ops._punica_C.dispatch_bgmv_low_level(
452
+ y,
453
+ x,
454
+ w_t_all,
455
+ indicies,
456
+ layer_idx,
457
+ scale,
458
+ h_in,
459
+ h_out,
460
+ y_offset,
461
+ )
vllm/attention/backends/flash_attn.py
+5
-5
@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
5
import torch
6
from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8
-from vllm._C import cache_ops
8
+from vllm import _custom_ops as ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata)
11
@@ -47,11 +47,11 @@ class FlashAttentionBackend(AttentionBackend):
47
) -> None:
48
src_key_cache = src_kv_cache[0]
49
dst_key_cache = dst_kv_cache[0]
50
- cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
50
+ ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
51
52
src_value_cache = src_kv_cache[1]
53
dst_value_cache = dst_kv_cache[1]
54
- cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
54
+ ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
55
56
@staticmethod
57
def copy_blocks(
@@ -60,7 +60,7 @@ class FlashAttentionBackend(AttentionBackend):
60
) -> None:
61
key_caches = [kv_cache[0] for kv_cache in kv_caches]
62
value_caches = [kv_cache[1] for kv_cache in kv_caches]
63
- cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
63
+ ops.copy_blocks(key_caches, value_caches, src_to_dists)
64
65
66
@dataclass
@@ -285,7 +285,7 @@ class FlashAttentionImpl(AttentionImpl):
285
# Reshape the input keys and values and store them in the cache.
286
# If kv_cache is not provided, the new key and value tensors are
287
# not cached. This happens during the initial memory profiling run.
288
- cache_ops.reshape_and_cache_flash(
288
+ ops.reshape_and_cache_flash(
289
key,
290
value,
291
key_cache,
vllm/distributed/device_communicators/custom_all_reduce.py
+19
-15
@@ -6,6 +6,7 @@ import torch.distributed as dist
6
from torch.distributed import ProcessGroup
7
8
import vllm.envs as envs
9
+from vllm import _custom_ops as ops
10
from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
from vllm.distributed.parallel_state import (
@@ -15,7 +16,11 @@ from vllm.logger import init_logger
16
try:
17
import pynvml
18
18
- from vllm._C import custom_ar
19
+ # Simulate ImportError if custom_ar ops are not supported.
20
+ if not ops.is_custom_op_supported("_C_custom_ar::meta_size"):
21
+ raise ImportError("custom_ar", __file__)
22
+
23
+ custom_ar = True
24
25
@contextmanager
26
def _nvml():
@@ -27,7 +32,7 @@ try:
32
33
except ImportError:
34
# For AMD GPUs
30
- custom_ar = None
35
+ custom_ar = False
36
pynvml = None
37
38
@contextmanager
@@ -97,7 +102,7 @@ class CustomAllreduce:
102
self._IS_CAPTURING = False
103
self.disabled = True
104
100
- if custom_ar is None:
105
+ if not custom_ar:
106
# disable because of missing custom allreduce library
107
# e.g. in a non-cuda environment
108
return
@@ -175,7 +180,7 @@ class CustomAllreduce:
180
# meta data composes of two parts: meta data for synchronization
181
# (256 bytes) and a temporary buffer for storing intermediate
182
# allreduce results.
178
- self.meta = torch.zeros(custom_ar.meta_size() + max_size,
183
+ self.meta = torch.zeros(ops.meta_size() + max_size,
184
dtype=torch.uint8,
185
device=self.device)
186
# This is a pre-registered IPC buffer. In eager mode, input tensors
@@ -196,9 +201,8 @@ class CustomAllreduce:
201
self.world_size = world_size
202
handles, offsets = self._get_ipc_meta(self.meta)
203
self.full_nvlink = full_nvlink
199
- self._ptr = custom_ar.init_custom_ar(self.meta, self.rank_data,
200
- handles, offsets, rank,
201
- self.full_nvlink)
204
+ self._ptr = ops.init_custom_ar(self.meta, self.rank_data, handles,
205
+ offsets, rank, self.full_nvlink)
206
self.register_buffer(self.buffer)
207
208
@contextmanager
@@ -252,31 +256,31 @@ class CustomAllreduce:
256
257
def register_buffer(self, inp: torch.Tensor):
258
handles, offsets = self._get_ipc_meta(inp)
255
- custom_ar.register_buffer(self._ptr, inp, handles, offsets)
259
+ ops.register_buffer(self._ptr, inp, handles, offsets)
260
261
def register_graph_buffers(self):
258
- handle, offset = custom_ar.get_graph_buffer_ipc_meta(self._ptr)
262
+ handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr)
263
handles, offsets = self._gather_ipc_meta((bytes(handle), offset))
264
logger.info("Registering %d cuda graph addresses", len(offset))
261
- custom_ar.register_graph_buffers(self._ptr, handles, offsets)
265
+ ops.register_graph_buffers(self._ptr, handles, offsets)
266
267
def should_custom_ar(self, inp: torch.Tensor):
264
- return custom_ar.should_custom_ar(inp, self.max_size, self.world_size,
265
- self.full_nvlink)
268
+ return ops.should_custom_ar(inp, self.max_size, self.world_size,
269
+ self.full_nvlink)
270
271
# all reduce, assuming inp tensor is IPC registered with register_buffer,
272
# or, in the context of cuda graphs, register_graph_buffers
273
def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None):
274
if out is None:
275
out = torch.empty_like(inp)
272
- custom_ar.all_reduce_reg(self._ptr, inp, out)
276
+ ops.all_reduce_reg(self._ptr, inp, out)
277
return out
278
279
# all reduce, assuming inp tensor is NOT IPC registered
280
def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None):
281
if out is None:
282
out = torch.empty_like(inp)
279
- custom_ar.all_reduce_unreg(self._ptr, inp, self.buffer, out)
283
+ ops.all_reduce_unreg(self._ptr, inp, self.buffer, out)
284
return out
285
286
def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
@@ -304,7 +308,7 @@ class CustomAllreduce:
308
309
def close(self):
310
if not self.disabled and self._ptr:
307
- custom_ar.dispose(self._ptr)
311
+ ops.dispose(self._ptr)
312
self._ptr = 0
313
314
def __del__(self):
vllm/lora/punica.py
+19
-26
@@ -4,16 +4,21 @@ from typing import Optional
4
5
import torch
6
7
+from vllm import _custom_ops as ops
8
+
9
+
10
+def _check_punica_support():
11
+ if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
12
+ return
13
8
-def _raise_import_error(e):
14
if torch.cuda.get_device_capability() < (8, 0):
15
raise ImportError(
11
- "punica LoRA kernels require compute capability >= 8.0") from e
16
+ "punica LoRA kernels require compute capability >= 8.0")
17
else:
18
raise ImportError(
19
"punica LoRA kernels could not be imported. If you built vLLM "
20
"from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
16
- "was set.") from e
21
+ "was set.")
22
23
24
def bgmv(
@@ -41,12 +46,9 @@ def bgmv(
46
layer_idx: Layer index of the weight matrices.
47
scale: Scaling factor.
48
"""
44
- try:
45
- import vllm._punica_C as punica_kernels
46
- except ImportError as e:
47
- _raise_import_error(e)
49
+ _check_punica_support()
50
49
- punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
51
+ ops.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
52
53
54
def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
@@ -75,11 +77,9 @@ def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
77
y_offset: Offset to apply to the starting column of y.
78
y_slice_size: Size of the y column slice.
79
"""
78
- try:
79
- import vllm._punica_C as punica_kernels
80
- except ImportError as e:
81
- _raise_import_error(e)
82
- punica_kernels.dispatch_bgmv_low_level(
80
+ _check_punica_support()
81
+
82
+ ops.dispatch_bgmv_low_level(
83
y,
84
x,
85
w_t_all,
@@ -122,10 +122,7 @@ def add_lora(y: torch.Tensor,
122
scale: Scaling factor.
123
buffer: Optional. Shape: `[B, R]`. Temporary buffer.
124
"""
125
- try:
126
- import vllm._punica_C as punica_kernels
127
- except ImportError as e:
128
- _raise_import_error(e)
125
+ _check_punica_support()
126
127
r = wb_t_all.size(-1)
128
if buffer is None:
@@ -135,9 +132,8 @@ def add_lora(y: torch.Tensor,
132
buffer = torch.zeros((x.size(0), r),
133
dtype=torch.float32,
134
device=x.device)
138
- punica_kernels.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
139
- punica_kernels.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx,
140
- scale)
135
+ ops.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
136
+ ops.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx, scale)
137
138
139
def add_lora_slice(y: torch.Tensor,
@@ -176,10 +172,7 @@ def add_lora_slice(y: torch.Tensor,
172
y_offset: Offset to apply to the starting column of y.
173
y_slice_size: Size of the y column slice.
174
"""
179
- try:
180
- import vllm._punica_C as punica_kernels
181
- except ImportError as e:
182
- _raise_import_error(e)
175
+ _check_punica_support()
176
177
r = wb_t_all.size(-1)
178
if buffer is None:
@@ -189,7 +182,7 @@ def add_lora_slice(y: torch.Tensor,
182
buffer = torch.zeros((x.size(0), r),
183
dtype=torch.float32,
184
device=x.device)
192
- punica_kernels.dispatch_bgmv_low_level(
185
+ ops.dispatch_bgmv_low_level(
186
buffer,
187
x,
188
wa_t_all,
@@ -200,7 +193,7 @@ def add_lora_slice(y: torch.Tensor,
193
buffer.size(1),
194
0,
195
)
203
- punica_kernels.dispatch_bgmv_low_level(
196
+ ops.dispatch_bgmv_low_level(
197
y,
198
buffer,
199
wb_t_all,
vllm/model_executor/layers/fused_moe/fused_moe.py
+1
-2
@@ -8,7 +8,6 @@ import torch
8
import triton
9
import triton.language as tl
10
11
-import vllm._moe_C as moe_kernels
11
from vllm import _custom_ops as ops
12
from vllm.logger import init_logger
13
@@ -355,7 +354,7 @@ def fused_topk(
354
topk,
355
dtype=torch.int32,
356
device=hidden_states.device)
358
- moe_kernels.topk_softmax(
357
+ ops.topk_softmax(
358
topk_weights,
359
topk_ids,
360
token_expert_indicies,
vllm/utils.py
+2
-5
@@ -22,6 +22,7 @@ import psutil
22
import torch
23
24
import vllm.envs as envs
25
+from vllm import _custom_ops as ops
26
from vllm.logger import enable_trace_function_call, init_logger
27
28
T = TypeVar("T")
@@ -148,12 +149,8 @@ def is_neuron() -> bool:
149
@lru_cache(maxsize=None)
150
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
151
"""Returns the maximum shared memory per thread block in bytes."""
151
- # NOTE: This import statement should be executed lazily since
152
- # the Neuron-X backend does not have the `cuda_utils` module.
153
- from vllm._C import cuda_utils
154
-
152
max_shared_mem = (
156
- cuda_utils.get_max_shared_memory_per_block_device_attribute(gpu))
153
+ ops.get_max_shared_memory_per_block_device_attribute(gpu))
154
# value 0 will cause MAX_SEQ_LEN become negative and test_attention.py
155
# will fail
156
assert max_shared_mem > 0, "max_shared_mem can not be zero"