@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f6d10c14c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Build] Enforce style for C++ and CUDA code with `clang-format` (#4722)

Michael Goin committed May 22, 2024 at 03:18 UTC 5f6d10c14c17122e6d711a4829ee0ca672e07f6f
64 files changed +6571 -6963
.clang-format new
+26
@@ -0,0 +1,26 @@
1 +BasedOnStyle: Google
2 +UseTab: Never
3 +IndentWidth: 2
4 +ColumnLimit: 80
5 +
6 +# Force pointers to the type for C++.
7 +DerivePointerAlignment: false
8 +PointerAlignment: Left
9 +
10 +# Reordering #include statements can (and currently will) introduce errors
11 +SortIncludes: false
12 +
13 +# Style choices
14 +AlignConsecutiveAssignments: false
15 +AlignConsecutiveDeclarations: false
16 +IndentPPDirectives: BeforeHash
17 +
18 +IncludeCategories:
19 + - Regex: '^<'
20 + Priority: 4
21 + - Regex: '^"(llvm|llvm-c|clang|clang-c|mlir|mlir-c)/'
22 + Priority: 3
23 + - Regex: '^"(qoda|\.\.)/'
24 + Priority: 2
25 + - Regex: '.*'
26 + Priority: 1
.github/workflows/clang-format.yml new
+42
@@ -0,0 +1,42 @@
1 +name: clang-format
2 +
3 +on:
4 + # Trigger the workflow on push or pull request,
5 + # but only for the main branch
6 + push:
7 + branches:
8 + - main
9 + pull_request:
10 + branches:
11 + - main
12 +
13 +jobs:
14 + clang-format:
15 + runs-on: ubuntu-latest
16 + strategy:
17 + matrix:
18 + python-version: ["3.11"]
19 + steps:
20 + - uses: actions/checkout@v2
21 + - name: Set up Python ${{ matrix.python-version }}
22 + uses: actions/setup-python@v2
23 + with:
24 + python-version: ${{ matrix.python-version }}
25 + - name: Install dependencies
26 + run: |
27 + python -m pip install --upgrade pip
28 + pip install clang-format==18.1.5
29 + - name: Running clang-format
30 + run: |
31 + EXCLUDES=(
32 + 'csrc/moe/topk_softmax_kernels.cu'
33 + 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
34 + 'csrc/punica/bgmv/bgmv_config.h'
35 + 'csrc/punica/bgmv/bgmv_impl.cuh'
36 + 'csrc/punica/bgmv/vec_dtypes.cuh'
37 + 'csrc/punica/punica_ops.cu'
38 + 'csrc/punica/type_convert.h'
39 + )
40 + find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
41 + | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
42 + | xargs clang-format --dry-run --Werror
\ No newline at end of file
csrc/activation_kernels.cu
+64 -75
@@ -10,11 +10,11 @@
10 namespace vllm {
11
12 // Activation and gating kernel template.
13 -template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
13 +template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
14 __global__ void act_and_mul_kernel(
15 - scalar_t* __restrict__ out, // [..., d]
16 - const scalar_t* __restrict__ input, // [..., 2, d]
17 - const int d) {
15 + scalar_t* __restrict__ out, // [..., d]
16 + const scalar_t* __restrict__ input, // [..., 2, d]
17 + const int d) {
18 const int64_t token_idx = blockIdx.x;
19 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
20 const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
@@ -23,72 +23,66 @@ __global__ void act_and_mul_kernel(
23 }
24 }
25
26 -template<typename T>
26 +template <typename T>
27 __device__ __forceinline__ T silu_kernel(const T& x) {
28 // x * sigmoid(x)
29 - return (T) (((float) x) / (1.0f + expf((float) -x)));
29 + return (T)(((float)x) / (1.0f + expf((float)-x)));
30 }
31
32 -template<typename T>
32 +template <typename T>
33 __device__ __forceinline__ T gelu_kernel(const T& x) {
34 // Equivalent to PyTorch GELU with 'none' approximation.
35 // Refer to:
36 // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
37 - const float f = (float) x;
37 + const float f = (float)x;
38 constexpr float ALPHA = M_SQRT1_2;
39 - return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
39 + return (T)(f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40 }
41
42 -template<typename T>
42 +template <typename T>
43 __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
44 // Equivalent to PyTorch GELU with 'tanh' approximation.
45 // Refer to:
46 // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
47 - const float f = (float) x;
47 + const float f = (float)x;
48 constexpr float BETA = M_SQRT2 * M_2_SQRTPI * 0.5f;
49 constexpr float KAPPA = 0.044715;
50 float x_cube = f * f * f;
51 float inner = BETA * (f + KAPPA * x_cube);
52 - return (T) (0.5f * f * (1.0f + ::tanhf(inner)));
52 + return (T)(0.5f * f * (1.0f + ::tanhf(inner)));
53 }
54
55 -} // namespace vllm
55 +} // namespace vllm
56
57 // Launch activation and gating kernel.
58 -#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
59 - int d = input.size(-1) / 2; \
60 - int64_t num_tokens = input.numel() / input.size(-1); \
61 - dim3 grid(num_tokens); \
62 - dim3 block(std::min(d, 1024)); \
63 - const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
64 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
65 - VLLM_DISPATCH_FLOATING_TYPES( \
66 - input.scalar_type(), \
67 - "act_and_mul_kernel", \
68 - [&] { \
69 - vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
70 - out.data_ptr<scalar_t>(), \
71 - input.data_ptr<scalar_t>(), \
72 - d); \
73 - });
74 -
75 -void silu_and_mul(
76 - torch::Tensor& out, // [..., d]
77 - torch::Tensor& input) // [..., 2 * d]
58 +#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
59 + int d = input.size(-1) / 2; \
60 + int64_t num_tokens = input.numel() / input.size(-1); \
61 + dim3 grid(num_tokens); \
62 + dim3 block(std::min(d, 1024)); \
63 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
64 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
65 + VLLM_DISPATCH_FLOATING_TYPES( \
66 + input.scalar_type(), "act_and_mul_kernel", [&] { \
67 + vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>> \
68 + <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
69 + input.data_ptr<scalar_t>(), d); \
70 + });
71 +
72 +void silu_and_mul(torch::Tensor& out, // [..., d]
73 + torch::Tensor& input) // [..., 2 * d]
74 {
75 LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
76 }
77
82 -void gelu_and_mul(
83 - torch::Tensor& out, // [..., d]
84 - torch::Tensor& input) // [..., 2 * d]
78 +void gelu_and_mul(torch::Tensor& out, // [..., d]
79 + torch::Tensor& input) // [..., 2 * d]
80 {
81 LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
82 }
83
89 -void gelu_tanh_and_mul(
90 - torch::Tensor& out, // [..., d]
91 - torch::Tensor& input) // [..., 2 * d]
84 +void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
85 + torch::Tensor& input) // [..., 2 * d]
86 {
87 LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
88 }
@@ -96,11 +90,11 @@ void gelu_tanh_and_mul(
90 namespace vllm {
91
92 // Element-wise activation kernel template.
99 -template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
93 +template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
94 __global__ void activation_kernel(
101 - scalar_t* __restrict__ out, // [..., d]
102 - const scalar_t* __restrict__ input, // [..., d]
103 - const int d) {
95 + scalar_t* __restrict__ out, // [..., d]
96 + const scalar_t* __restrict__ input, // [..., d]
97 + const int d) {
98 const int64_t token_idx = blockIdx.x;
99 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
100 const scalar_t x = VLLM_LDG(&input[token_idx * d + idx]);
@@ -108,54 +102,49 @@ __global__ void activation_kernel(
102 }
103 }
104
111 -} // namespace vllm
105 +} // namespace vllm
106
107 // Launch element-wise activation kernel.
114 -#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
115 - int d = input.size(-1); \
116 - int64_t num_tokens = input.numel() / d; \
117 - dim3 grid(num_tokens); \
118 - dim3 block(std::min(d, 1024)); \
119 - const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
120 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
121 - VLLM_DISPATCH_FLOATING_TYPES( \
122 - input.scalar_type(), \
123 - "activation_kernel", \
124 - [&] { \
125 - vllm::activation_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
126 - out.data_ptr<scalar_t>(), \
127 - input.data_ptr<scalar_t>(), \
128 - d); \
129 - });
108 +#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
109 + int d = input.size(-1); \
110 + int64_t num_tokens = input.numel() / d; \
111 + dim3 grid(num_tokens); \
112 + dim3 block(std::min(d, 1024)); \
113 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
114 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
115 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "activation_kernel", [&] { \
116 + vllm::activation_kernel<scalar_t, KERNEL<scalar_t>> \
117 + <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
118 + input.data_ptr<scalar_t>(), d); \
119 + });
120
121 namespace vllm {
122
133 -template<typename T>
123 +template <typename T>
124 __device__ __forceinline__ T gelu_new_kernel(const T& x) {
135 - const float x3 = (float) (x * x * x);
136 - const T t = (T) tanhf((T) (0.79788456f * (float) (x + (T) (0.044715f * x3))));
137 - return ((T) 0.5) * x * (((T) 1.0) + t);
125 + const float x3 = (float)(x * x * x);
126 + const T t = (T)tanhf((T)(0.79788456f * (float)(x + (T)(0.044715f * x3))));
127 + return ((T)0.5) * x * (((T)1.0) + t);
128 }
129
140 -template<typename T>
130 +template <typename T>
131 __device__ __forceinline__ T gelu_fast_kernel(const T& x) {
142 - const float f = (float) x;
143 - const T t = (T) tanhf(((T) (f * 0.79788456f)) * (((T) 1.0) + (T) (0.044715f * f) * x));
144 - return ((T) 0.5) * x * (((T) 1.0) + t);
132 + const float f = (float)x;
133 + const T t =
134 + (T)tanhf(((T)(f * 0.79788456f)) * (((T)1.0) + (T)(0.044715f * f) * x));
135 + return ((T)0.5) * x * (((T)1.0) + t);
136 }
137
147 -} // namespace vllm
138 +} // namespace vllm
139
149 -void gelu_new(
150 - torch::Tensor& out, // [..., d]
151 - torch::Tensor& input) // [..., d]
140 +void gelu_new(torch::Tensor& out, // [..., d]
141 + torch::Tensor& input) // [..., d]
142 {
143 LAUNCH_ACTIVATION_KERNEL(vllm::gelu_new_kernel);
144 }
145
156 -void gelu_fast(
157 - torch::Tensor& out, // [..., d]
158 - torch::Tensor& input) // [..., d]
146 +void gelu_fast(torch::Tensor& out, // [..., d]
147 + torch::Tensor& input) // [..., d]
148 {
149 LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
150 }
csrc/attention/attention_generic.cuh
+10 -9
@@ -1,5 +1,6 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4 * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
@@ -22,31 +23,31 @@
23 namespace vllm {
24
25 // A vector type to store Q, K, V elements.
25 -template<typename T, int VEC_SIZE>
26 +template <typename T, int VEC_SIZE>
27 struct Vec {};
28
29 // A vector type to store FP32 accumulators.
29 -template<typename T>
30 +template <typename T>
31 struct FloatVec {};
32
33 // Template vector operations.
33 -template<typename Acc, typename A, typename B>
34 +template <typename Acc, typename A, typename B>
35 inline __device__ Acc mul(A a, B b);
36
36 -template<typename T>
37 +template <typename T>
38 inline __device__ float sum(T v);
39
39 -template<typename T>
40 +template <typename T>
41 inline __device__ float dot(T a, T b) {
42 return sum(mul<T, T, T>(a, b));
43 }
44
44 -template<typename A, typename T>
45 +template <typename A, typename T>
46 inline __device__ float dot(T a, T b) {
47 return sum(mul<A, T, T>(a, b));
48 }
49
49 -template<typename T>
50 +template <typename T>
51 inline __device__ void zero(T& dst) {
52 constexpr int WORDS = sizeof(T) / 4;
53 union {
@@ -61,4 +62,4 @@ inline __device__ void zero(T& dst) {
62 dst = tmp.raw;
63 }
64
64 -} // namespace vllm
65 +} // namespace vllm
csrc/attention/attention_kernels.cu
+299 -337
@@ -1,5 +1,6 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
@@ -27,15 +28,15 @@
28 #ifdef USE_ROCM
29 #include <hip/hip_bf16.h>
30 #include "../quantization/fp8/amd/quant_utils.cuh"
30 - typedef __hip_bfloat16 __nv_bfloat16;
31 +typedef __hip_bfloat16 __nv_bfloat16;
32 #else
33 #include "../quantization/fp8/nvidia/quant_utils.cuh"
34 #endif
35
36 #ifndef USE_ROCM
36 -#define WARP_SIZE 32
37 + #define WARP_SIZE 32
38 #else
38 -#define WARP_SIZE warpSize
39 + #define WARP_SIZE warpSize
40 #endif
41
42 #define MAX(a, b) ((a) > (b) ? (a) : (b))
@@ -45,7 +46,7 @@
46 namespace vllm {
47
48 // Utility function for attention softmax.
48 -template<int NUM_WARPS>
49 +template <int NUM_WARPS>
50 inline __device__ float block_sum(float* red_smem, float sum) {
51 // Decompose the thread index into warp / lane.
52 int warp = threadIdx.x / WARP_SIZE;
@@ -82,31 +83,28 @@ inline __device__ float block_sum(float* red_smem, float sum) {
83
84 // TODO(woosuk): Merge the last two dimensions of the grid.
85 // Grid: (num_heads, num_seqs, max_num_partitions).
85 -template<
86 - typename scalar_t,
87 - typename cache_t,
88 - int HEAD_SIZE,
89 - int BLOCK_SIZE,
90 - int NUM_THREADS,
91 - vllm::Fp8KVCacheDataType KV_DTYPE,
92 - int PARTITION_SIZE = 0> // Zero means no partitioning.
86 +template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
87 + int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
88 + int PARTITION_SIZE = 0> // Zero means no partitioning.
89 __device__ void paged_attention_kernel(
94 - float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
95 - float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
96 - scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
97 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
98 - const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
99 - const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
100 - const int num_kv_heads, // [num_heads]
101 - const float scale,
102 - const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
103 - const int* __restrict__ seq_lens, // [num_seqs]
104 - const int max_num_blocks_per_seq,
105 - const float* __restrict__ alibi_slopes, // [num_heads]
106 - const int q_stride,
107 - const int kv_block_stride,
108 - const int kv_head_stride,
109 - const float kv_scale) {
90 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
91 + float* __restrict__ max_logits, // [num_seqs, num_heads,
92 + // max_num_partitions]
93 + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
94 + // head_size]
95 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
96 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
97 + // head_size/x, block_size, x]
98 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
99 + // head_size, block_size]
100 + const int num_kv_heads, // [num_heads]
101 + const float scale,
102 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
103 + const int* __restrict__ seq_lens, // [num_seqs]
104 + const int max_num_blocks_per_seq,
105 + const float* __restrict__ alibi_slopes, // [num_heads]
106 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
107 + const float kv_scale) {
108 const int seq_idx = blockIdx.y;
109 const int partition_idx = blockIdx.z;
110 const int max_num_partitions = gridDim.z;
@@ -118,22 +116,29 @@ __device__ void paged_attention_kernel(
116 }
117
118 const int num_seq_blocks = DIVIDE_ROUND_UP(seq_len, BLOCK_SIZE);
121 - const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
119 + const int num_blocks_per_partition =
120 + USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
121
122 // [start_block_idx, end_block_idx) is the range of blocks to process.
124 - const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
125 - const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
123 + const int start_block_idx =
124 + USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
125 + const int end_block_idx =
126 + MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
127 const int num_blocks = end_block_idx - start_block_idx;
128
129 // [start_token_idx, end_token_idx) is the range of tokens to process.
130 const int start_token_idx = start_block_idx * BLOCK_SIZE;
130 - const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
131 + const int end_token_idx =
132 + MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
133 const int num_tokens = end_token_idx - start_token_idx;
134
135 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
134 - constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
136 + constexpr int NUM_THREAD_GROUPS =
137 + NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE
138 + // divides NUM_THREADS
139 assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
136 - constexpr int NUM_TOKENS_PER_THREAD_GROUP = DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
140 + constexpr int NUM_TOKENS_PER_THREAD_GROUP =
141 + DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
142 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
143 const int thread_idx = threadIdx.x;
144 const int warp_idx = thread_idx / WARP_SIZE;
@@ -143,13 +148,14 @@ __device__ void paged_attention_kernel(
148 const int num_heads = gridDim.x;
149 const int num_queries_per_kv = num_heads / num_kv_heads;
150 const int kv_head_idx = head_idx / num_queries_per_kv;
146 - const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
151 + const float alibi_slope =
152 + alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
153
154 // A vector type to store a part of a key or a query.
149 - // The vector size is configured in such a way that the threads in a thread group
150 - // fetch or compute 16 bytes at a time.
151 - // For example, if the size of a thread group is 4 and the data type is half,
152 - // then the vector size is 16 / (4 * sizeof(half)) == 2.
155 + // The vector size is configured in such a way that the threads in a thread
156 + // group fetch or compute 16 bytes at a time. For example, if the size of a
157 + // thread group is 4 and the data type is half, then the vector size is 16 /
158 + // (4 * sizeof(half)) == 2.
159 constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
160 using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
161 using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
@@ -163,18 +169,21 @@ __device__ void paged_attention_kernel(
169
170 // Load the query to registers.
171 // Each thread in a thread group has a different part of the query.
166 - // For example, if the the thread group size is 4, then the first thread in the group
167 - // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
168 - // th vectors of the query, and so on.
169 - // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
172 + // For example, if the the thread group size is 4, then the first thread in
173 + // the group has 0, 4, 8, ... th vectors of the query, and the second thread
174 + // has 1, 5, 9, ... th vectors of the query, and so on. NOTE(woosuk): Because
175 + // q is split from a qkv tensor, it may not be contiguous.
176 const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
177 __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
178 #pragma unroll
173 - for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) {
179 + for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD;
180 + i += NUM_THREAD_GROUPS) {
181 const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
175 - q_vecs[thread_group_offset][i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
182 + q_vecs[thread_group_offset][i] =
183 + *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
184 }
177 - __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a memory wall right before we use q_vecs
185 + __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a
186 + // memory wall right before we use q_vecs
187
188 // Memory planning.
189 extern __shared__ char shared_mem[];
@@ -193,44 +202,50 @@ __device__ void paged_attention_kernel(
202 // Each thread group in a warp fetches a key from the block, and computes
203 // dot product with the query.
204 const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
196 - for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
197 - // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
198 - // because int32 can lead to overflow when this variable is multiplied by large numbers
199 - // (e.g., kv_block_stride).
200 - const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
205 + for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
206 + block_idx += NUM_WARPS) {
207 + // NOTE(woosuk): The block number is stored in int32. However, we cast it to
208 + // int64 because int32 can lead to overflow when this variable is multiplied
209 + // by large numbers (e.g., kv_block_stride).
210 + const int64_t physical_block_number =
211 + static_cast<int64_t>(block_table[block_idx]);
212
213 // Load a key to registers.
214 // Each thread in a thread group has a different part of the key.
204 - // For example, if the the thread group size is 4, then the first thread in the group
205 - // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
206 - // vectors of the key, and so on.
215 + // For example, if the the thread group size is 4, then the first thread in
216 + // the group has 0, 4, 8, ... th vectors of the key, and the second thread
217 + // has 1, 5, 9, ... th vectors of the key, and so on.
218 for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
208 - const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
219 + const int physical_block_offset =
220 + (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
221 const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
222 K_vec k_vecs[NUM_VECS_PER_THREAD];
223
224 #pragma unroll
225 for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
214 - const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride
215 - + kv_head_idx * kv_head_stride
216 - + physical_block_offset * x;
226 + const cache_t* k_ptr =
227 + k_cache + physical_block_number * kv_block_stride +
228 + kv_head_idx * kv_head_stride + physical_block_offset * x;
229 const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
230 const int offset1 = (vec_idx * VEC_SIZE) / x;
231 const int offset2 = (vec_idx * VEC_SIZE) % x;
232
233 if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
222 - k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
234 + k_vecs[j] = *reinterpret_cast<const K_vec*>(
235 + k_ptr + offset1 * BLOCK_SIZE * x + offset2);
236 } else {
237 // Vector conversion from Quant_vec to K_vec.
238 Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
226 - k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227 - k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(k_vec_quant, kv_scale);
239 + k_ptr + offset1 * BLOCK_SIZE * x + offset2);
240 + k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
241 + k_vec_quant, kv_scale);
242 }
243 }
244
245 // Compute dot product.
246 // This includes a reduction across the threads in the same thread group.
233 - float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
247 + float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(
248 + q_vecs[thread_group_offset], k_vecs);
249 // Add the ALiBi bias if slopes are given.
250 qk += (alibi_slope != 0) ? alibi_slope * (token_idx - seq_len + 1) : 0;
251
@@ -285,13 +300,12 @@ __device__ void paged_attention_kernel(
300
301 // If partitioning is enabled, store the max logit and exp_sum.
302 if (USE_PARTITIONING && thread_idx == 0) {
288 - float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
289 - + head_idx * max_num_partitions
290 - + partition_idx;
303 + float* max_logits_ptr = max_logits +
304 + seq_idx * num_heads * max_num_partitions +
305 + head_idx * max_num_partitions + partition_idx;
306 *max_logits_ptr = qk_max;
292 - float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
293 - + head_idx * max_num_partitions
294 - + partition_idx;
307 + float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions +
308 + head_idx * max_num_partitions + partition_idx;
309 *exp_sums_ptr = exp_sum;
310 }
311
@@ -304,7 +318,8 @@ __device__ void paged_attention_kernel(
318
319 constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
320 constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
307 - constexpr int NUM_ROWS_PER_THREAD = DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
321 + constexpr int NUM_ROWS_PER_THREAD =
322 + DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
323
324 // NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
325 float accs[NUM_ROWS_PER_THREAD];
@@ -315,18 +330,21 @@ __device__ void paged_attention_kernel(
330
331 scalar_t zero_value;
332 zero(zero_value);
318 - for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
319 - // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
320 - // because int32 can lead to overflow when this variable is multiplied by large numbers
321 - // (e.g., kv_block_stride).
322 - const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
333 + for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
334 + block_idx += NUM_WARPS) {
335 + // NOTE(woosuk): The block number is stored in int32. However, we cast it to
336 + // int64 because int32 can lead to overflow when this variable is multiplied
337 + // by large numbers (e.g., kv_block_stride).
338 + const int64_t physical_block_number =
339 + static_cast<int64_t>(block_table[block_idx]);
340 const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
341 const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
342 L_vec logits_vec;
326 - from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
343 + from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx -
344 + start_token_idx));
345
328 - const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride
329 - + kv_head_idx * kv_head_stride;
346 + const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride +
347 + kv_head_idx * kv_head_stride;
348 #pragma unroll
349 for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
350 const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -337,14 +355,17 @@ __device__ void paged_attention_kernel(
355 if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
356 v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
357 } else {
340 - V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358 + V_quant_vec v_quant_vec =
359 + *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
360 // Vector conversion from V_quant_vec to V_vec.
342 - v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec, kv_scale);
361 + v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
362 + kv_scale);
363 }
364 if (block_idx == num_seq_blocks - 1) {
345 - // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
346 - // we should explicitly zero out the values since they may contain NaNs.
347 - // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
365 + // NOTE(woosuk): When v_vec contains the tokens that are out of the
366 + // context, we should explicitly zero out the values since they may
367 + // contain NaNs. See
368 + // https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
369 scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
370 #pragma unroll
371 for (int j = 0; j < V_VEC_SIZE; j++) {
@@ -367,8 +388,8 @@ __device__ void paged_attention_kernel(
388 accs[i] = acc;
389 }
390
370 - // NOTE(woosuk): A barrier is required because the shared memory space for logits
371 - // is reused for the output.
391 + // NOTE(woosuk): A barrier is required because the shared memory space for
392 + // logits is reused for the output.
393 __syncthreads();
394
395 // Perform reduction across warps.
@@ -405,9 +426,9 @@ __device__ void paged_attention_kernel(
426
427 // Write the final output.
428 if (warp_idx == 0) {
408 - scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
409 - + head_idx * max_num_partitions * HEAD_SIZE
410 - + partition_idx * HEAD_SIZE;
429 + scalar_t* out_ptr =
430 + out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
431 + head_idx * max_num_partitions * HEAD_SIZE + partition_idx * HEAD_SIZE;
432 #pragma unroll
433 for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
434 const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -419,79 +440,75 @@ __device__ void paged_attention_kernel(
440 }
441
442 // Grid: (num_heads, num_seqs, 1).
422 -template<
423 - typename scalar_t,
424 - typename cache_t,
425 - int HEAD_SIZE,
426 - int BLOCK_SIZE,
427 - int NUM_THREADS,
428 - vllm::Fp8KVCacheDataType KV_DTYPE>
443 +template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
444 + int NUM_THREADS,
445 + vllm::Fp8KVCacheDataType KV_DTYPE>
446 __global__ void paged_attention_v1_kernel(
430 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
431 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
432 - const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
433 - const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
434 - const int num_kv_heads, // [num_heads]
435 - const float scale,
436 - const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
437 - const int* __restrict__ seq_lens, // [num_seqs]
438 - const int max_num_blocks_per_seq,
439 - const float* __restrict__ alibi_slopes, // [num_heads]
440 - const int q_stride,
441 - const int kv_block_stride,
442 - const int kv_head_stride,
443 - const float kv_scale) {
444 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>(
445 - /* exp_sums */ nullptr, /* max_logits */ nullptr,
446 - out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
447 - max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
447 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
448 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
449 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
450 + // head_size/x, block_size, x]
451 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
452 + // head_size, block_size]
453 + const int num_kv_heads, // [num_heads]
454 + const float scale,
455 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
456 + const int* __restrict__ seq_lens, // [num_seqs]
457 + const int max_num_blocks_per_seq,
458 + const float* __restrict__ alibi_slopes, // [num_heads]
459 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
460 + const float kv_scale) {
461 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
462 + KV_DTYPE>(
463 + /* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
464 + v_cache, num_kv_heads, scale, block_tables, seq_lens,
465 + max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
466 + kv_head_stride, kv_scale);
467 }
468
469 // Grid: (num_heads, num_seqs, max_num_partitions).
451 -template<
452 - typename scalar_t,
453 - typename cache_t,
454 - int HEAD_SIZE,
455 - int BLOCK_SIZE,
456 - int NUM_THREADS,
457 - vllm::Fp8KVCacheDataType KV_DTYPE,
458 - int PARTITION_SIZE>
470 +template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
471 + int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
472 + int PARTITION_SIZE>
473 __global__ void paged_attention_v2_kernel(
460 - float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
461 - float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
462 - scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
463 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
464 - const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
465 - const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
466 - const int num_kv_heads, // [num_heads]
467 - const float scale,
468 - const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
469 - const int* __restrict__ seq_lens, // [num_seqs]
470 - const int max_num_blocks_per_seq,
471 - const float* __restrict__ alibi_slopes, // [num_heads]
472 - const int q_stride,
473 - const int kv_block_stride,
474 - const int kv_head_stride,
475 - const float kv_scale) {
476 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE, PARTITION_SIZE>(
477 - exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
478 - block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
479 - q_stride, kv_block_stride, kv_head_stride, kv_scale);
474 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
475 + float* __restrict__ max_logits, // [num_seqs, num_heads,
476 + // max_num_partitions]
477 + scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
478 + // max_num_partitions, head_size]
479 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
480 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
481 + // head_size/x, block_size, x]
482 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
483 + // head_size, block_size]
484 + const int num_kv_heads, // [num_heads]
485 + const float scale,
486 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
487 + const int* __restrict__ seq_lens, // [num_seqs]
488 + const int max_num_blocks_per_seq,
489 + const float* __restrict__ alibi_slopes, // [num_heads]
490 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
491 + const float kv_scale) {
492 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
493 + KV_DTYPE, PARTITION_SIZE>(
494 + exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
495 + block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
496 + kv_block_stride, kv_head_stride, kv_scale);
497 }
498
499 // Grid: (num_heads, num_seqs).
483 -template<
484 - typename scalar_t,
485 - int HEAD_SIZE,
486 - int NUM_THREADS,
487 - int PARTITION_SIZE>
500 +template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
501 + int PARTITION_SIZE>
502 __global__ void paged_attention_v2_reduce_kernel(
489 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
490 - const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
491 - const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
492 - const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
493 - const int* __restrict__ seq_lens, // [num_seqs]
494 - const int max_num_partitions) {
503 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
504 + const float* __restrict__ exp_sums, // [num_seqs, num_heads,
505 + // max_num_partitions]
506 + const float* __restrict__ max_logits, // [num_seqs, num_heads,
507 + // max_num_partitions]
508 + const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
509 + // max_num_partitions, head_size]
510 + const int* __restrict__ seq_lens, // [num_seqs]
511 + const int max_num_partitions) {
512 const int num_heads = gridDim.x;
513 const int head_idx = blockIdx.x;
514 const int seq_idx = blockIdx.y;
@@ -499,9 +516,11 @@ __global__ void paged_attention_v2_reduce_kernel(
516 const int num_partitions = DIVIDE_ROUND_UP(seq_len, PARTITION_SIZE);
517 if (num_partitions == 1) {
518 // No need to reduce. Only copy tmp_out to out.
502 - scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
503 - const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
504 - + head_idx * max_num_partitions * HEAD_SIZE;
519 + scalar_t* out_ptr =
520 + out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
521 + const scalar_t* tmp_out_ptr =
522 + tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
523 + head_idx * max_num_partitions * HEAD_SIZE;
524 for (int i = threadIdx.x; i < HEAD_SIZE; i += blockDim.x) {
525 out_ptr[i] = tmp_out_ptr[i];
526 }
@@ -520,8 +539,9 @@ __global__ void paged_attention_v2_reduce_kernel(
539
540 // Load max logits to shared memory.
541 float* shared_max_logits = reinterpret_cast<float*>(shared_mem);
523 - const float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
524 - + head_idx * max_num_partitions;
542 + const float* max_logits_ptr = max_logits +
543 + seq_idx * num_heads * max_num_partitions +
544 + head_idx * max_num_partitions;
545 float max_logit = -FLT_MAX;
546 for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
547 const float l = max_logits_ptr[i];
@@ -550,9 +570,11 @@ __global__ void paged_attention_v2_reduce_kernel(
570 max_logit = VLLM_SHFL_SYNC(max_logit, 0);
571
572 // Load rescaled exp sums to shared memory.
553 - float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
554 - const float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
555 - + head_idx * max_num_partitions;
573 + float* shared_exp_sums =
574 + reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
575 + const float* exp_sums_ptr = exp_sums +
576 + seq_idx * num_heads * max_num_partitions +
577 + head_idx * max_num_partitions;
578 float global_exp_sum = 0.0f;
579 for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
580 float l = shared_max_logits[i];
@@ -565,61 +587,45 @@ __global__ void paged_attention_v2_reduce_kernel(
587 const float inv_global_exp_sum = __fdividef(1.0f, global_exp_sum + 1e-6f);
588
589 // Aggregate tmp_out to out.
568 - const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
569 - + head_idx * max_num_partitions * HEAD_SIZE;
570 - scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
590 + const scalar_t* tmp_out_ptr =
591 + tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
592 + head_idx * max_num_partitions * HEAD_SIZE;
593 + scalar_t* out_ptr =
594 + out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
595 #pragma unroll
596 for (int i = threadIdx.x; i < HEAD_SIZE; i += NUM_THREADS) {
597 float acc = 0.0f;
598 for (int j = 0; j < num_partitions; ++j) {
575 - acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] * inv_global_exp_sum;
599 + acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] *
600 + inv_global_exp_sum;
601 }
602 from_float(out_ptr[i], acc);
603 }
604 }
605
581 -} // namespace vllm
582 -
583 -#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
584 - VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
585 - ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
586 - KV_DTYPE>), shared_mem_size); \
587 - vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
588 - KV_DTYPE><<<grid, block, shared_mem_size, stream>>>( \
589 - out_ptr, \
590 - query_ptr, \
591 - key_cache_ptr, \
592 - value_cache_ptr, \
593 - num_kv_heads, \
594 - scale, \
595 - block_tables_ptr, \
596 - seq_lens_ptr, \
597 - max_num_blocks_per_seq, \
598 - alibi_slopes_ptr, \
599 - q_stride, \
600 - kv_block_stride, \
601 - kv_head_stride, \
602 - kv_scale);
606 +} // namespace vllm
607 +
608 +#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
609 + VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
610 + ((void*)vllm::paged_attention_v1_kernel< \
611 + T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>), \
612 + shared_mem_size); \
613 + vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
614 + NUM_THREADS, KV_DTYPE> \
615 + <<<grid, block, shared_mem_size, stream>>>( \
616 + out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
617 + scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
618 + alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
619 + kv_scale);
620
621 // TODO(woosuk): Tune NUM_THREADS.
605 -template<
606 - typename T,
607 - typename CACHE_T,
608 - int BLOCK_SIZE,
609 - vllm::Fp8KVCacheDataType KV_DTYPE,
610 - int NUM_THREADS = 128>
622 +template <typename T, typename CACHE_T, int BLOCK_SIZE,
623 + vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128>
624 void paged_attention_v1_launcher(
612 - torch::Tensor& out,
613 - torch::Tensor& query,
614 - torch::Tensor& key_cache,
615 - torch::Tensor& value_cache,
616 - int num_kv_heads,
617 - float scale,
618 - torch::Tensor& block_tables,
619 - torch::Tensor& seq_lens,
620 - int max_seq_len,
621 - const c10::optional<torch::Tensor>& alibi_slopes,
622 - float kv_scale) {
625 + torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
626 + torch::Tensor& value_cache, int num_kv_heads, float scale,
627 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
628 + const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
629 int num_seqs = query.size(0);
630 int num_heads = query.size(1);
631 int head_size = query.size(2);
@@ -632,9 +638,10 @@ void paged_attention_v1_launcher(
638 assert(head_size % thread_group_size == 0);
639
640 // NOTE: alibi_slopes is optional.
635 - const float* alibi_slopes_ptr = alibi_slopes ?
636 - reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
637 - : nullptr;
641 + const float* alibi_slopes_ptr =
642 + alibi_slopes
643 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
644 + : nullptr;
645
646 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
647 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
@@ -644,7 +651,8 @@ void paged_attention_v1_launcher(
651 int* seq_lens_ptr = seq_lens.data_ptr<int>();
652
653 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
647 - int padded_max_seq_len = DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
654 + int padded_max_seq_len =
655 + DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
656 int logits_size = padded_max_seq_len * sizeof(float);
657 int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
658 // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
@@ -683,19 +691,10 @@ void paged_attention_v1_launcher(
691 }
692 }
693
686 -#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
687 - paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
688 - out, \
689 - query, \
690 - key_cache, \
691 - value_cache, \
692 - num_kv_heads, \
693 - scale, \
694 - block_tables, \
695 - seq_lens, \
696 - max_seq_len, \
697 - alibi_slopes, \
698 - kv_scale);
694 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
695 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
696 + out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
697 + seq_lens, max_seq_len, alibi_slopes, kv_scale);
698
699 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
700 // 1, 2, 4, 64, 128, 256.
@@ -716,74 +715,45 @@ void paged_attention_v1_launcher(
715 }
716
717 void paged_attention_v1(
719 - torch::Tensor& out, // [num_seqs, num_heads, head_size]
720 - torch::Tensor& query, // [num_seqs, num_heads, head_size]
721 - torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
722 - torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
723 - int num_kv_heads, // [num_heads]
724 - float scale,
725 - torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
726 - torch::Tensor& seq_lens, // [num_seqs]
727 - int block_size,
728 - int max_seq_len,
729 - const c10::optional<torch::Tensor>& alibi_slopes,
730 - const std::string& kv_cache_dtype,
731 - float kv_scale) {
732 -
733 - DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V1_LAUNCHER_BLOCK_SIZE)
734 -}
735 -
736 -#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
737 - vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
738 - KV_DTYPE, PARTITION_SIZE> \
739 - <<<grid, block, shared_mem_size, stream>>>( \
740 - exp_sums_ptr, \
741 - max_logits_ptr, \
742 - tmp_out_ptr, \
743 - query_ptr, \
744 - key_cache_ptr, \
745 - value_cache_ptr, \
746 - num_kv_heads, \
747 - scale, \
748 - block_tables_ptr, \
749 - seq_lens_ptr, \
750 - max_num_blocks_per_seq, \
751 - alibi_slopes_ptr, \
752 - q_stride, \
753 - kv_block_stride, \
754 - kv_head_stride, \
755 - kv_scale); \
756 - vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
757 - <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
758 - out_ptr, \
759 - exp_sums_ptr, \
760 - max_logits_ptr, \
761 - tmp_out_ptr, \
762 - seq_lens_ptr, \
763 - max_num_partitions);
764 -
765 -template<
766 - typename T,
767 - typename CACHE_T,
768 - int BLOCK_SIZE,
769 - vllm::Fp8KVCacheDataType KV_DTYPE,
770 - int NUM_THREADS = 128,
771 - int PARTITION_SIZE = 512>
718 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
719 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
720 + torch::Tensor&
721 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
722 + torch::Tensor&
723 + value_cache, // [num_blocks, num_heads, head_size, block_size]
724 + int num_kv_heads, // [num_heads]
725 + float scale,
726 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
727 + torch::Tensor& seq_lens, // [num_seqs]
728 + int block_size, int max_seq_len,
729 + const c10::optional<torch::Tensor>& alibi_slopes,
730 + const std::string& kv_cache_dtype, float kv_scale){
731 +
732 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
733 + CALL_V1_LAUNCHER_BLOCK_SIZE)}
734 +#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
735 + vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
736 + NUM_THREADS, KV_DTYPE, PARTITION_SIZE> \
737 + <<<grid, block, shared_mem_size, stream>>>( \
738 + exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
739 + value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
740 + seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
741 + kv_block_stride, kv_head_stride, kv_scale); \
742 + vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
743 + PARTITION_SIZE> \
744 + <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
745 + out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
746 + max_num_partitions);
747 +
748 +template <typename T, typename CACHE_T, int BLOCK_SIZE,
749 + vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128,
750 + int PARTITION_SIZE = 512>
751 void paged_attention_v2_launcher(
773 - torch::Tensor& out,
774 - torch::Tensor& exp_sums,
775 - torch::Tensor& max_logits,
776 - torch::Tensor& tmp_out,
777 - torch::Tensor& query,
778 - torch::Tensor& key_cache,
779 - torch::Tensor& value_cache,
780 - int num_kv_heads,
781 - float scale,
782 - torch::Tensor& block_tables,
783 - torch::Tensor& seq_lens,
784 - int max_seq_len,
785 - const c10::optional<torch::Tensor>& alibi_slopes,
786 - float kv_scale) {
752 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
753 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
754 + torch::Tensor& value_cache, int num_kv_heads, float scale,
755 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
756 + const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
757 int num_seqs = query.size(0);
758 int num_heads = query.size(1);
759 int head_size = query.size(2);
@@ -796,9 +766,10 @@ void paged_attention_v2_launcher(
766 assert(head_size % thread_group_size == 0);
767
768 // NOTE: alibi_slopes is optional.
799 - const float* alibi_slopes_ptr = alibi_slopes ?
800 - reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
801 - : nullptr;
769 + const float* alibi_slopes_ptr =
770 + alibi_slopes
771 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
772 + : nullptr;
773
774 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
775 float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
@@ -853,59 +824,50 @@ void paged_attention_v2_launcher(
824 }
825 }
826
856 -#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
857 - paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
858 - out, \
859 - exp_sums, \
860 - max_logits, \
861 - tmp_out, \
862 - query, \
863 - key_cache, \
864 - value_cache, \
865 - num_kv_heads, \
866 - scale, \
867 - block_tables, \
868 - seq_lens, \
869 - max_seq_len, \
870 - alibi_slopes, \
871 - kv_scale);
827 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
828 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
829 + out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
830 + num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
831 + kv_scale);
832
833 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
834 // 1, 2, 4, 64, 128, 256.
875 -#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
876 - switch (block_size) { \
877 - case 8: \
878 - CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
879 - break; \
880 - case 16: \
881 - CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
882 - break; \
883 - case 32: \
884 - CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
885 - break; \
886 - default: \
887 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
888 - break; \
835 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
836 + switch (block_size) { \
837 + case 8: \
838 + CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
839 + break; \
840 + case 16: \
841 + CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
842 + break; \
843 + case 32: \
844 + CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
845 + break; \
846 + default: \
847 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
848 + break; \
849 }
850
851 void paged_attention_v2(
892 - torch::Tensor& out, // [num_seqs, num_heads, head_size]
893 - torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
894 - torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
895 - torch::Tensor& tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
896 - torch::Tensor& query, // [num_seqs, num_heads, head_size]
897 - torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
898 - torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
899 - int num_kv_heads, // [num_heads]
900 - float scale,
901 - torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
902 - torch::Tensor& seq_lens, // [num_seqs]
903 - int block_size,
904 - int max_seq_len,
905 - const c10::optional<torch::Tensor>& alibi_slopes,
906 - const std::string& kv_cache_dtype,
907 - float kv_scale) {
908 - DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V2_LAUNCHER_BLOCK_SIZE)
852 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
853 + torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
854 + torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
855 + torch::Tensor&
856 + tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
857 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
858 + torch::Tensor&
859 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
860 + torch::Tensor&
861 + value_cache, // [num_blocks, num_heads, head_size, block_size]
862 + int num_kv_heads, // [num_heads]
863 + float scale,
864 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
865 + torch::Tensor& seq_lens, // [num_seqs]
866 + int block_size, int max_seq_len,
867 + const c10::optional<torch::Tensor>& alibi_slopes,
868 + const std::string& kv_cache_dtype, float kv_scale) {
869 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
870 + CALL_V2_LAUNCHER_BLOCK_SIZE)
871 }
872
873 #undef WARP_SIZE
csrc/attention/attention_utils.cuh
+6 -5
@@ -1,5 +1,6 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
@@ -26,7 +27,7 @@
27 namespace vllm {
28
29 // Q*K^T operation.
29 -template<int THREAD_GROUP_SIZE, typename Vec, int N>
30 +template <int THREAD_GROUP_SIZE, typename Vec, int N>
31 inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
32 using A_vec = typename FloatVec<Vec>::Type;
33 // Compute the parallel products for Q*K^T (treat vector lanes separately).
@@ -45,12 +46,12 @@ inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
46 return qk;
47 }
48
48 -template<typename T, int THREAD_GROUP_SIZE>
49 +template <typename T, int THREAD_GROUP_SIZE>
50 struct Qk_dot {
50 - template<typename Vec, int N>
51 + template <typename Vec, int N>
52 static inline __device__ float dot(const Vec (&q)[N], const Vec (&k)[N]) {
53 return qk_dot_<THREAD_GROUP_SIZE>(q, k);
54 }
55 };
56
56 -} // namespace vllm
57 +} // namespace vllm
csrc/attention/dtype_bfloat16.cuh
+39 -35
@@ -1,6 +1,8 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 - * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 + * and
5 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6 * Copyright (c) 2023, The vLLM team.
7 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8 *
@@ -28,8 +30,8 @@
30 #include <hip/hip_bf16.h>
31 #include <hip/hip_fp16.h>
32
31 - typedef __hip_bfloat162 __nv_bfloat162;
32 - typedef __hip_bfloat16 __nv_bfloat16;
33 +typedef __hip_bfloat162 __nv_bfloat162;
34 +typedef __hip_bfloat16 __nv_bfloat16;
35 #endif
36
37 #include <stdint.h>
@@ -50,37 +52,37 @@ struct bf16_8_t {
52 };
53
54 // BF16 vector types for Q, K, V.
53 -template<>
55 +template <>
56 struct Vec<__nv_bfloat16, 1> {
57 using Type = __nv_bfloat16;
58 };
57 -template<>
59 +template <>
60 struct Vec<__nv_bfloat16, 2> {
61 using Type = __nv_bfloat162;
62 };
61 -template<>
63 +template <>
64 struct Vec<__nv_bfloat16, 4> {
65 using Type = bf16_4_t;
66 };
65 -template<>
67 +template <>
68 struct Vec<__nv_bfloat16, 8> {
69 using Type = bf16_8_t;
70 };
71
72 // FP32 accumulator vector types corresponding to Vec.
71 -template<>
73 +template <>
74 struct FloatVec<__nv_bfloat16> {
75 using Type = float;
76 };
75 -template<>
77 +template <>
78 struct FloatVec<__nv_bfloat162> {
79 using Type = float2;
80 };
79 -template<>
81 +template <>
82 struct FloatVec<bf16_4_t> {
83 using Type = Float4_;
84 };
83 -template<>
85 +template <>
86 struct FloatVec<bf16_8_t> {
87 using Type = Float8_;
88 };
@@ -108,9 +110,9 @@ inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
110 assert(false);
111 #else
112 #ifndef USE_ROCM
111 - return a + b;
113 + return a + b;
114 #else
113 - return __hadd(a, b);
115 + return __hadd(a, b);
116 #endif
117 #endif
118 }
@@ -161,7 +163,7 @@ inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
163 }
164
165 // Vector multiplication.
164 -template<>
166 +template <>
167 inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
168 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
169 assert(false);
@@ -170,7 +172,7 @@ inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
172 #endif
173 }
174
173 -template<>
175 +template <>
176 inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
177 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
178 assert(false);
@@ -179,12 +181,12 @@ inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
181 #endif
182 }
183
182 -template<>
184 +template <>
185 inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
186 return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
187 }
188
187 -template<>
189 +template <>
190 inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
191 bf16_4_t c;
192 c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -192,7 +194,7 @@ inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
194 return c;
195 }
196
195 -template<>
197 +template <>
198 inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
199 __nv_bfloat162 s = bf162bf162(a);
200 bf16_4_t c;
@@ -201,7 +203,7 @@ inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
203 return c;
204 }
205
204 -template<>
206 +template <>
207 inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
208 bf16_8_t c;
209 c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -211,7 +213,7 @@ inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
213 return c;
214 }
215
214 -template<>
216 +template <>
217 inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
218 __nv_bfloat162 s = bf162bf162(a);
219 bf16_8_t c;
@@ -222,26 +224,26 @@ inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
224 return c;
225 }
226
225 -template<>
227 +template <>
228 inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b) {
229 float fa = __bfloat162float(a);
230 float fb = __bfloat162float(b);
231 return fa * fb;
232 }
233
232 -template<>
234 +template <>
235 inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
236 float2 fa = bf1622float2(a);
237 float2 fb = bf1622float2(b);
238 return mul<float2, float2, float2>(fa, fb);
239 }
240
239 -template<>
241 +template <>
242 inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
243 return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
244 }
245
244 -template<>
246 +template <>
247 inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
248 Float4_ fc;
249 fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -249,7 +251,7 @@ inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
251 return fc;
252 }
253
252 -template<>
254 +template <>
255 inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
256 __nv_bfloat162 s = bf162bf162(a);
257 Float4_ fc;
@@ -258,7 +260,7 @@ inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
260 return fc;
261 }
262
261 -template<>
263 +template <>
264 inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
265 Float8_ fc;
266 fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -268,7 +270,7 @@ inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
270 return fc;
271 }
272
271 -template<>
273 +template <>
274 inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
275 __nv_bfloat162 s = bf162bf162(a);
276 Float8_ fc;
@@ -280,7 +282,8 @@ inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
282 }
283
284 // Vector fused multiply-add.
283 -inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
285 +inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b,
286 + __nv_bfloat162 c) {
287 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
288 assert(false);
289 #else
@@ -288,7 +291,8 @@ inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bf
291 #endif
292 }
293
291 -inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
294 +inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b,
295 + __nv_bfloat162 c) {
296 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
297 assert(false);
298 #else
@@ -379,23 +383,23 @@ inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc) {
383 }
384
385 // Vector sum.
382 -template<>
386 +template <>
387 inline __device__ float sum(__nv_bfloat16 v) {
388 return __bfloat162float(v);
389 }
390
387 -template<>
391 +template <>
392 inline __device__ float sum(__nv_bfloat162 v) {
393 float2 vf = bf1622float2(v);
394 return vf.x + vf.y;
395 }
396
393 -template<>
397 +template <>
398 inline __device__ float sum(bf16_4_t v) {
399 return sum(v.x) + sum(v.y);
400 }
401
398 -template<>
402 +template <>
403 inline __device__ float sum(bf16_8_t v) {
404 return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
405 }
@@ -448,4 +452,4 @@ inline __device__ void zero(__nv_bfloat16& dst) {
452 #endif
453 }
454
451 -} // namespace vllm
455 +} // namespace vllm
csrc/attention/dtype_float16.cuh
+47 -45
@@ -1,6 +1,8 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 - * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 + * and
5 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6 * Copyright (c) 2023, The vLLM team.
7 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8 *
@@ -30,37 +32,37 @@
32 namespace vllm {
33
34 // FP16 vector types for Q, K, V.
33 -template<>
35 +template <>
36 struct Vec<uint16_t, 1> {
37 using Type = uint16_t;
38 };
37 -template<>
39 +template <>
40 struct Vec<uint16_t, 2> {
41 using Type = uint32_t;
42 };
41 -template<>
43 +template <>
44 struct Vec<uint16_t, 4> {
45 using Type = uint2;
46 };
45 -template<>
47 +template <>
48 struct Vec<uint16_t, 8> {
49 using Type = uint4;
50 };
51
52 // FP32 accumulator vector types corresponding to Vec.
51 -template<>
53 +template <>
54 struct FloatVec<uint16_t> {
55 using Type = float;
56 };
55 -template<>
57 +template <>
58 struct FloatVec<uint32_t> {
59 using Type = float2;
60 };
59 -template<>
61 +template <>
62 struct FloatVec<uint2> {
63 using Type = Float4_;
64 };
63 -template<>
65 +template <>
66 struct FloatVec<uint4> {
67 using Type = Float8_;
68 };
@@ -73,8 +75,8 @@ inline __device__ uint32_t h0_h0(uint16_t a) {
75 return b;
76 #else
77 union {
76 - uint32_t u32;
77 - uint16_t u16[2];
78 + uint32_t u32;
79 + uint16_t u16[2];
80 } tmp;
81 tmp.u16[0] = a;
82 tmp.u16[1] = a;
@@ -130,10 +132,12 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
132 } tmp;
133 #ifndef USE_ROCM
134 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
133 - asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
135 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n"
136 + : "=r"(tmp.u32)
137 + : "f"(f.y), "f"(f.x));
138 #else
135 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
136 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
139 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
140 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
141 #endif
142 #else
143 tmp.u16[0] = float_to_half(f.x);
@@ -201,7 +205,7 @@ inline __device__ Float8_ add(uint4 a, Float8_ fb) {
205 }
206
207 // Vector multiplication.
204 -template<>
208 +template <>
209 inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
210 uint16_t c;
211 #ifndef USE_ROCM
@@ -212,7 +216,7 @@ inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
216 return c;
217 }
218
215 -template<>
219 +template <>
220 inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
221 uint32_t c;
222 #ifndef USE_ROCM
@@ -223,12 +227,12 @@ inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
227 return c;
228 }
229
226 -template<>
230 +template <>
231 inline __device__ uint32_t mul(uint16_t a, uint32_t b) {
232 return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
233 }
234
231 -template<>
235 +template <>
236 inline __device__ uint2 mul(uint2 a, uint2 b) {
237 uint2 c;
238 c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
@@ -236,7 +240,7 @@ inline __device__ uint2 mul(uint2 a, uint2 b) {
240 return c;
241 }
242
239 -template<>
243 +template <>
244 inline __device__ uint2 mul(uint16_t a, uint2 b) {
245 uint32_t s = h0_h0(a);
246 uint2 c;
@@ -245,7 +249,7 @@ inline __device__ uint2 mul(uint16_t a, uint2 b) {
249 return c;
250 }
251
248 -template<>
252 +template <>
253 inline __device__ uint4 mul(uint4 a, uint4 b) {
254 uint4 c;
255 c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
@@ -255,7 +259,7 @@ inline __device__ uint4 mul(uint4 a, uint4 b) {
259 return c;
260 }
261
258 -template<>
262 +template <>
263 inline __device__ uint4 mul(uint16_t a, uint4 b) {
264 uint32_t s = h0_h0(a);
265 uint4 c;
@@ -266,26 +270,26 @@ inline __device__ uint4 mul(uint16_t a, uint4 b) {
270 return c;
271 }
272
269 -template<>
273 +template <>
274 inline __device__ float mul(uint16_t a, uint16_t b) {
275 float fa = half_to_float(a);
276 float fb = half_to_float(b);
277 return fa * fb;
278 }
279
276 -template<>
280 +template <>
281 inline __device__ float2 mul(uint32_t a, uint32_t b) {
282 float2 fa = half2_to_float2(a);
283 float2 fb = half2_to_float2(b);
284 return mul<float2, float2, float2>(fa, fb);
285 }
286
283 -template<>
287 +template <>
288 inline __device__ float2 mul(uint16_t a, uint32_t b) {
289 return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
290 }
291
288 -template<>
292 +template <>
293 inline __device__ Float4_ mul(uint2 a, uint2 b) {
294 Float4_ fc;
295 fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
@@ -293,7 +297,7 @@ inline __device__ Float4_ mul(uint2 a, uint2 b) {
297 return fc;
298 }
299
296 -template<>
300 +template <>
301 inline __device__ Float4_ mul(uint16_t a, uint2 b) {
302 uint32_t s = h0_h0(a);
303 Float4_ fc;
@@ -302,7 +306,7 @@ inline __device__ Float4_ mul(uint16_t a, uint2 b) {
306 return fc;
307 }
308
305 -template<>
309 +template <>
310 inline __device__ Float8_ mul(uint4 a, uint4 b) {
311 Float8_ fc;
312 fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
@@ -312,7 +316,7 @@ inline __device__ Float8_ mul(uint4 a, uint4 b) {
316 return fc;
317 }
318
315 -template<>
319 +template <>
320 inline __device__ Float8_ mul(uint16_t a, uint4 b) {
321 uint32_t s = h0_h0(a);
322 Float8_ fc;
@@ -327,9 +331,13 @@ inline __device__ Float8_ mul(uint16_t a, uint4 b) {
331 inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
332 uint32_t d;
333 #ifndef USE_ROCM
330 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
334 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
335 + : "=r"(d)
336 + : "r"(a), "r"(b), "r"(c));
337 #else
332 - asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n" : "=v"(d) : "v"(a), "v"(b), "v"(c));
338 + asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n"
339 + : "=v"(d)
340 + : "v"(a), "v"(b), "v"(c));
341 #endif
342 return d;
343 }
@@ -423,24 +431,24 @@ inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc) {
431 }
432
433 // Vector sum.
426 -template<>
434 +template <>
435 inline __device__ float sum(uint16_t v) {
436 return half_to_float(v);
437 }
438
431 -template<>
439 +template <>
440 inline __device__ float sum(uint32_t v) {
441 float2 tmp = half2_to_float2(v);
442 return tmp.x + tmp.y;
443 }
444
437 -template<>
445 +template <>
446 inline __device__ float sum(uint2 v) {
447 uint32_t c = add(v.x, v.y);
448 return sum(c);
449 }
450
443 -template<>
451 +template <>
452 inline __device__ float sum(uint4 v) {
453 uint32_t c = add(v.x, v.y);
454 c = add(c, v.z);
@@ -470,13 +478,9 @@ inline __device__ void from_float(uint4& dst, Float8_ src) {
478 }
479
480 // From float16 to float32.
473 -inline __device__ float to_float(uint16_t u) {
474 - return half_to_float(u);
475 -}
481 +inline __device__ float to_float(uint16_t u) { return half_to_float(u); }
482
477 -inline __device__ float2 to_float(uint32_t u) {
478 - return half2_to_float2(u);
479 -}
483 +inline __device__ float2 to_float(uint32_t u) { return half2_to_float2(u); }
484
485 inline __device__ Float4_ to_float(uint2 u) {
486 Float4_ tmp;
@@ -495,8 +499,6 @@ inline __device__ Float8_ to_float(uint4 u) {
499 }
500
501 // Zero-out a variable.
498 -inline __device__ void zero(uint16_t& dst) {
499 - dst = uint16_t(0);
500 -}
502 +inline __device__ void zero(uint16_t& dst) { dst = uint16_t(0); }
503
502 -} // namespace vllm
504 +} // namespace vllm
csrc/attention/dtype_float32.cuh
+33 -55
@@ -1,6 +1,8 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3 - * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 + * and
5 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6 * Copyright (c) 2023, The vLLM team.
7 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8 *
@@ -38,37 +40,35 @@ struct Float8_ {
40 };
41
42 // FP32 vector types for Q, K, V.
41 -template<>
43 +template <>
44 struct Vec<float, 1> {
45 using Type = float;
46 };
45 -template<>
47 +template <>
48 struct Vec<float, 2> {
49 using Type = float2;
50 };
49 -template<>
51 +template <>
52 struct Vec<float, 4> {
53 using Type = float4;
54 };
55
56 // FP32 accumulator vector types corresponding to Vec.
55 -template<>
57 +template <>
58 struct FloatVec<float> {
59 using Type = float;
60 };
59 -template<>
61 +template <>
62 struct FloatVec<float2> {
63 using Type = float2;
64 };
63 -template<>
65 +template <>
66 struct FloatVec<float4> {
67 using Type = float4;
68 };
69
70 // Vector addition.
69 -inline __device__ float add(float a, float b) {
70 - return a + b;
71 -}
71 +inline __device__ float add(float a, float b) { return a + b; }
72
73 inline __device__ float2 add(float2 a, float2 b) {
74 float2 c;
@@ -87,12 +87,12 @@ inline __device__ float4 add(float4 a, float4 b) {
87 }
88
89 // Vector multiplication.
90 -template<>
90 +template <>
91 inline __device__ float mul<float, float>(float a, float b) {
92 return a * b;
93 }
94
95 -template<>
95 +template <>
96 inline __device__ float2 mul(float2 a, float2 b) {
97 float2 c;
98 c.x = a.x * b.x;
@@ -100,7 +100,7 @@ inline __device__ float2 mul(float2 a, float2 b) {
100 return c;
101 }
102
103 -template<>
103 +template <>
104 inline __device__ float2 mul(float a, float2 b) {
105 float2 c;
106 c.x = a * b.x;
@@ -108,7 +108,7 @@ inline __device__ float2 mul(float a, float2 b) {
108 return c;
109 }
110
111 -template<>
111 +template <>
112 inline __device__ float4 mul(float4 a, float4 b) {
113 float4 c;
114 c.x = a.x * b.x;
@@ -118,7 +118,7 @@ inline __device__ float4 mul(float4 a, float4 b) {
118 return c;
119 }
120
121 -template<>
121 +template <>
122 inline __device__ float4 mul(float a, float4 b) {
123 float4 c;
124 c.x = a * b.x;
@@ -129,9 +129,7 @@ inline __device__ float4 mul(float a, float4 b) {
129 }
130
131 // Vector fused multiply-add.
132 -inline __device__ float fma(float a, float b, float c) {
133 - return a * b + c;
134 -}
132 +inline __device__ float fma(float a, float b, float c) { return a * b + c; }
133
134 inline __device__ float2 fma(float2 a, float2 b, float2 c) {
135 float2 d;
@@ -182,35 +180,33 @@ inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c) {
180 }
181
182 // Vector sum.
185 -template<>
183 +template <>
184 inline __device__ float sum(float v) {
185 return v;
186 }
187
190 -template<>
188 +template <>
189 inline __device__ float sum(float2 v) {
190 return v.x + v.y;
191 }
192
195 -template<>
193 +template <>
194 inline __device__ float sum(float4 v) {
195 return v.x + v.y + v.z + v.w;
196 }
197
200 -template<>
198 +template <>
199 inline __device__ float sum(Float4_ v) {
200 return v.x.x + v.x.y + v.y.x + v.y.y;
201 }
202
205 -template<>
203 +template <>
204 inline __device__ float sum(Float8_ v) {
205 return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
206 }
207
208 // Vector dot product.
211 -inline __device__ float dot(float a, float b) {
212 - return a * b;
213 -}
209 +inline __device__ float dot(float a, float b) { return a * b; }
210
211 inline __device__ float dot(float2 a, float2 b) {
212 float2 c = mul<float2, float2, float2>(a, b);
@@ -232,42 +228,24 @@ inline __device__ float dot(Float8_ a, Float8_ b) {
228 }
229
230 // From float to float.
235 -inline __device__ void from_float(float& dst, float src) {
236 - dst = src;
237 -}
231 +inline __device__ void from_float(float& dst, float src) { dst = src; }
232
239 -inline __device__ void from_float(float2& dst, float2 src) {
240 - dst = src;
241 -}
233 +inline __device__ void from_float(float2& dst, float2 src) { dst = src; }
234
243 -inline __device__ void from_float(float4& dst, float4 src) {
244 - dst = src;
245 -}
235 +inline __device__ void from_float(float4& dst, float4 src) { dst = src; }
236
237 // From float to float.
248 -inline __device__ float to_float(float u) {
249 - return u;
250 -}
238 +inline __device__ float to_float(float u) { return u; }
239
252 -inline __device__ float2 to_float(float2 u) {
253 - return u;
254 -}
240 +inline __device__ float2 to_float(float2 u) { return u; }
241
256 -inline __device__ float4 to_float(float4 u) {
257 - return u;
258 -}
242 +inline __device__ float4 to_float(float4 u) { return u; }
243
260 -inline __device__ Float4_ to_float(Float4_ u) {
261 - return u;
262 -}
244 +inline __device__ Float4_ to_float(Float4_ u) { return u; }
245
264 -inline __device__ Float8_ to_float(Float8_ u) {
265 - return u;
266 -}
246 +inline __device__ Float8_ to_float(Float8_ u) { return u; }
247
248 // Zero-out a variable.
269 -inline __device__ void zero(float& dst) {
270 - dst = 0.f;
271 -}
249 +inline __device__ void zero(float& dst) { dst = 0.f; }
250
273 -} // namespace vllm
251 +} // namespace vllm
csrc/attention/dtype_fp8.cuh
+16 -16
@@ -4,38 +4,38 @@
4
5 #include <stdint.h>
6 #ifdef ENABLE_FP8
7 -#ifndef USE_ROCM
8 -#include <cuda_fp8.h>
9 -#endif // USE_ROCM
10 -#endif // ENABLE_FP8
7 + #ifndef USE_ROCM
8 + #include <cuda_fp8.h>
9 + #endif // USE_ROCM
10 +#endif // ENABLE_FP8
11
12 namespace vllm {
13
14 enum class Fp8KVCacheDataType {
15 - kAuto = 0,
16 - kFp8E4M3 = 1,
17 - kFp8E5M2 = 2,
15 + kAuto = 0,
16 + kFp8E4M3 = 1,
17 + kFp8E5M2 = 2,
18 };
19
20 // fp8 vector types for quantization of kv cache
21 -template<>
21 +template <>
22 struct Vec<uint8_t, 1> {
23 - using Type = uint8_t;
23 + using Type = uint8_t;
24 };
25
26 -template<>
26 +template <>
27 struct Vec<uint8_t, 2> {
28 - using Type = uint16_t;
28 + using Type = uint16_t;
29 };
30
31 -template<>
31 +template <>
32 struct Vec<uint8_t, 4> {
33 - using Type = uint32_t;
33 + using Type = uint32_t;
34 };
35
36 -template<>
36 +template <>
37 struct Vec<uint8_t, 8> {
38 - using Type = uint2;
38 + using Type = uint2;
39 };
40
41 -} // namespace vllm
41 +} // namespace vllm
csrc/cache.h
+16 -28
@@ -5,36 +5,24 @@
5 #include <map>
6 #include <vector>
7
8 -void swap_blocks(
9 - torch::Tensor& src,
10 - torch::Tensor& dst,
11 - const torch::Tensor& block_mapping);
8 +void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
9 + const torch::Tensor& block_mapping);
10
13 -void copy_blocks(
14 - std::vector<torch::Tensor>& key_caches,
15 - std::vector<torch::Tensor>& value_caches,
16 - const torch::Tensor& block_mapping);
11 +void copy_blocks(std::vector<torch::Tensor>& key_caches,
12 + std::vector<torch::Tensor>& value_caches,
13 + const torch::Tensor& block_mapping);
14
18 -void reshape_and_cache(
19 - torch::Tensor& key,
20 - torch::Tensor& value,
21 - torch::Tensor& key_cache,
22 - torch::Tensor& value_cache,
23 - torch::Tensor& slot_mapping,
24 - const std::string& kv_cache_dtype,
25 - const float kv_scale);
15 +void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
16 + torch::Tensor& key_cache, torch::Tensor& value_cache,
17 + torch::Tensor& slot_mapping,
18 + const std::string& kv_cache_dtype, const float kv_scale);
19
27 -void reshape_and_cache_flash(
28 - torch::Tensor& key,
29 - torch::Tensor& value,
30 - torch::Tensor& key_cache,
31 - torch::Tensor& value_cache,
32 - torch::Tensor& slot_mapping,
33 - const std::string& kv_cache_dtype);
20 +void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
21 + torch::Tensor& key_cache,
22 + torch::Tensor& value_cache,
23 + torch::Tensor& slot_mapping,
24 + const std::string& kv_cache_dtype);
25
26 // Just for unittest
36 -void convert_fp8(
37 - torch::Tensor& dst_cache,
38 - torch::Tensor& src_cache,
39 - const float scale,
40 - const std::string& kv_cache_dtype);
27 +void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
28 + const float scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+131 -157
@@ -6,9 +6,9 @@
6 #include "dispatch_utils.h"
7
8 #ifdef USE_ROCM
9 -#include "quantization/fp8/amd/quant_utils.cuh"
9 + #include "quantization/fp8/amd/quant_utils.cuh"
10 #else
11 -#include "quantization/fp8/nvidia/quant_utils.cuh"
11 + #include "quantization/fp8/nvidia/quant_utils.cuh"
12 #endif
13
14 #include <algorithm>
@@ -18,20 +18,17 @@
18
19 #ifdef USE_ROCM
20 #include <hip/hip_bf16.h>
21 - typedef __hip_bfloat16 __nv_bfloat16;
21 +typedef __hip_bfloat16 __nv_bfloat16;
22 #endif
23
24 -void swap_blocks(
25 - torch::Tensor& src,
26 - torch::Tensor& dst,
27 - const torch::Tensor& block_mapping) {
24 +void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
25 + const torch::Tensor& block_mapping) {
26 torch::Device src_device = src.device();
27 torch::Device dst_device = dst.device();
28 cudaMemcpyKind memcpy_type;
29 if (src_device.is_cuda() && dst_device.is_cuda()) {
32 - TORCH_CHECK(
33 - src_device.index() == dst_device.index(),
34 - "src and dst must be on the same GPU");
30 + TORCH_CHECK(src_device.index() == dst_device.index(),
31 + "src and dst must be on the same GPU");
32 memcpy_type = cudaMemcpyDeviceToDevice;
33 } else if (src_device.is_cuda() && dst_device.is_cpu()) {
34 memcpy_type = cudaMemcpyDeviceToHost;
@@ -41,16 +38,17 @@ void swap_blocks(
38 TORCH_CHECK(false, "Invalid device combination");
39 }
40
44 - // NOTE(youkaichao): keep in mind that `block_mapping` should be
41 + // NOTE(youkaichao): keep in mind that `block_mapping` should be
42 // a cpu tensor, otherwise every `item` call will require a gpu-cpu
43 // synchronization.
44 TORCH_CHECK(block_mapping.device().is_cpu(), "block_mapping must be on CPU");
45
49 - char *src_ptr = static_cast<char*>(src.data_ptr());
50 - char *dst_ptr = static_cast<char*>(dst.data_ptr());
46 + char* src_ptr = static_cast<char*>(src.data_ptr());
47 + char* dst_ptr = static_cast<char*>(dst.data_ptr());
48
49 const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
53 - const at::cuda::OptionalCUDAGuard device_guard(src_device.is_cuda() ? src_device : dst_device);
50 + const at::cuda::OptionalCUDAGuard device_guard(
51 + src_device.is_cuda() ? src_device : dst_device);
52 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
53 // NOTE(woosuk): This can be slow if the number of blocks is large.
54 const int64_t num_blocks = block_mapping.size(0);
@@ -59,29 +57,25 @@ void swap_blocks(
57 int64_t dst_block_number = block_mapping[i][1].item<int64_t>();
58 int64_t src_offset = src_block_number * block_size_in_bytes;
59 int64_t dst_offset = dst_block_number * block_size_in_bytes;
62 - cudaMemcpyAsync(
63 - dst_ptr + dst_offset,
64 - src_ptr + src_offset,
65 - block_size_in_bytes,
66 - memcpy_type,
67 - stream);
60 + cudaMemcpyAsync(dst_ptr + dst_offset, src_ptr + src_offset,
61 + block_size_in_bytes, memcpy_type, stream);
62 }
63 }
64
65 namespace vllm {
66
67 // Grid: (num_layers, num_pairs)
74 -template<typename scalar_t>
75 -__global__ void copy_blocks_kernel(
76 - int64_t* key_cache_ptrs,
77 - int64_t* value_cache_ptrs,
78 - const int64_t* __restrict__ block_mapping,
79 - const int numel_per_block) {
68 +template <typename scalar_t>
69 +__global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
70 + int64_t* value_cache_ptrs,
71 + const int64_t* __restrict__ block_mapping,
72 + const int numel_per_block) {
73 const int layer_idx = blockIdx.x;
74 const int pair_idx = blockIdx.y;
75
76 scalar_t* key_cache = reinterpret_cast<scalar_t*>(key_cache_ptrs[layer_idx]);
84 - scalar_t* value_cache = reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
77 + scalar_t* value_cache =
78 + reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
79 int64_t src_block_number = block_mapping[2 * pair_idx];
80 int64_t dst_block_number = block_mapping[2 * pair_idx + 1];
81
@@ -99,12 +93,11 @@ __global__ void copy_blocks_kernel(
93 }
94 }
95
102 -} // namespace vllm
96 +} // namespace vllm
97
104 -void copy_blocks(
105 - std::vector<torch::Tensor>& key_caches,
106 - std::vector<torch::Tensor>& value_caches,
107 - const torch::Tensor& block_mapping) {
98 +void copy_blocks(std::vector<torch::Tensor>& key_caches,
99 + std::vector<torch::Tensor>& value_caches,
100 + const torch::Tensor& block_mapping) {
101 int num_layers = key_caches.size();
102 TORCH_CHECK(num_layers == value_caches.size());
103 if (num_layers == 0) {
@@ -118,8 +111,10 @@ void copy_blocks(
111 int64_t key_cache_ptrs[num_layers];
112 int64_t value_cache_ptrs[num_layers];
113 for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
121 - key_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
122 - value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
114 + key_cache_ptrs[layer_idx] =
115 + reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
116 + value_cache_ptrs[layer_idx] =
117 + reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
118 }
119
120 // block_mapping is a 2D tensor with shape (num_pairs, 2).
@@ -127,10 +122,12 @@ void copy_blocks(
122
123 // Move the data structures to the GPU.
124 // NOTE: This synchronizes the CPU and GPU.
130 - torch::Tensor key_cache_ptrs_tensor = torch::from_blob(
131 - key_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
132 - torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
133 - value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
125 + torch::Tensor key_cache_ptrs_tensor =
126 + torch::from_blob(key_cache_ptrs, {num_layers}, torch::kInt64)
127 + .to(cache_device);
128 + torch::Tensor value_cache_ptrs_tensor =
129 + torch::from_blob(value_cache_ptrs, {num_layers}, torch::kInt64)
130 + .to(cache_device);
131
132 // Launch the kernel.
133 const int numel_per_block = key_caches[0][0].numel();
@@ -139,31 +136,28 @@ void copy_blocks(
136 const at::cuda::OptionalCUDAGuard device_guard(cache_device);
137 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
138 VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
142 - key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
143 - vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
144 - key_cache_ptrs_tensor.data_ptr<int64_t>(),
145 - value_cache_ptrs_tensor.data_ptr<int64_t>(),
146 - block_mapping.data_ptr<int64_t>(),
147 - numel_per_block);
148 - }));
139 + key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
140 + vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
141 + key_cache_ptrs_tensor.data_ptr<int64_t>(),
142 + value_cache_ptrs_tensor.data_ptr<int64_t>(),
143 + block_mapping.data_ptr<int64_t>(), numel_per_block);
144 + }));
145 }
146
147 namespace vllm {
148
153 -template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
149 +template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
150 __global__ void reshape_and_cache_kernel(
155 - const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
156 - const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
157 - cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
158 - cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
159 - const int64_t* __restrict__ slot_mapping, // [num_tokens]
160 - const int key_stride,
161 - const int value_stride,
162 - const int num_heads,
163 - const int head_size,
164 - const int block_size,
165 - const int x,
166 - const float kv_scale) {
151 + const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
152 + const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
153 + cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x,
154 + // block_size, x]
155 + cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size,
156 + // block_size]
157 + const int64_t* __restrict__ slot_mapping, // [num_tokens]
158 + const int key_stride, const int value_stride, const int num_heads,
159 + const int head_size, const int block_size, const int x,
160 + const float kv_scale) {
161 const int64_t token_idx = blockIdx.x;
162 const int64_t slot_idx = slot_mapping[token_idx];
163 if (slot_idx < 0) {
@@ -184,40 +178,39 @@ __global__ void reshape_and_cache_kernel(
178 const int x_idx = head_offset / x;
179 const int x_offset = head_offset % x;
180
187 - const int64_t tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
188 - + head_idx * (head_size / x) * block_size * x
189 - + x_idx * block_size * x
190 - + block_offset * x
191 - + x_offset;
192 - const int64_t tgt_value_idx = block_idx * num_heads * head_size * block_size
193 - + head_idx * head_size * block_size
194 - + head_offset * block_size
195 - + block_offset;
181 + const int64_t tgt_key_idx =
182 + block_idx * num_heads * (head_size / x) * block_size * x +
183 + head_idx * (head_size / x) * block_size * x + x_idx * block_size * x +
184 + block_offset * x + x_offset;
185 + const int64_t tgt_value_idx =
186 + block_idx * num_heads * head_size * block_size +
187 + head_idx * head_size * block_size + head_offset * block_size +
188 + block_offset;
189 scalar_t tgt_key = key[src_key_idx];
190 scalar_t tgt_value = value[src_value_idx];
191 if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
192 key_cache[tgt_key_idx] = tgt_key;
193 value_cache[tgt_value_idx] = tgt_value;
194 } else {
202 - key_cache[tgt_key_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
203 - value_cache[tgt_value_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
195 + key_cache[tgt_key_idx] =
196 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
197 + value_cache[tgt_value_idx] =
198 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
199 }
200 }
201 }
202
208 -template<typename scalar_t>
203 +template <typename scalar_t>
204 __global__ void reshape_and_cache_flash_kernel(
210 - const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
211 - const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
212 - scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads, head_size]
213 - scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads, head_size]
214 - const int64_t* __restrict__ slot_mapping, // [num_tokens]
215 - const int block_stride,
216 - const int key_stride,
217 - const int value_stride,
218 - const int num_heads,
219 - const int head_size,
220 - const int block_size) {
205 + const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
206 + const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
207 + scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads,
208 + // head_size]
209 + scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads,
210 + // head_size]
211 + const int64_t* __restrict__ slot_mapping, // [num_tokens]
212 + const int block_stride, const int key_stride, const int value_stride,
213 + const int num_heads, const int head_size, const int block_size) {
214 const int64_t token_idx = blockIdx.x;
215 const int64_t slot_idx = slot_mapping[token_idx];
216 // NOTE: slot_idx can be -1 if the token is padded
@@ -232,43 +225,37 @@ __global__ void reshape_and_cache_flash_kernel(
225 const int64_t src_value_idx = token_idx * value_stride + i;
226 const int head_idx = i / head_size;
227 const int head_offset = i % head_size;
235 - const int64_t tgt_value_idx = block_idx * block_stride
236 - + block_offset * num_heads * head_size
237 - + head_idx * head_size
238 - + head_offset;
228 + const int64_t tgt_value_idx = block_idx * block_stride +
229 + block_offset * num_heads * head_size +
230 + head_idx * head_size + head_offset;
231 k_cache[tgt_value_idx] = key[src_key_idx];
232 v_cache[tgt_value_idx] = value[src_value_idx];
233 }
234 }
243 -} // namespace vllm
235 +} // namespace vllm
236
237 // KV_T is the stored data type of kv-cache.
238 // CACHE_T is the data type of key and value tensors.
239 // KV_DTYPE is the real data type of kv-cache.
248 -#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
249 - vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE><<<grid, block, 0, stream>>>( \
250 - reinterpret_cast<KV_T*>(key.data_ptr()), \
251 - reinterpret_cast<KV_T*>(value.data_ptr()), \
252 - reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
253 - reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
254 - slot_mapping.data_ptr<int64_t>(), \
255 - key_stride, \
256 - value_stride, \
257 - num_heads, \
258 - head_size, \
259 - block_size, \
260 - x, \
261 - kv_scale);
240 +#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
241 + vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE> \
242 + <<<grid, block, 0, stream>>>( \
243 + reinterpret_cast<KV_T*>(key.data_ptr()), \
244 + reinterpret_cast<KV_T*>(value.data_ptr()), \
245 + reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
246 + reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
247 + slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
248 + num_heads, head_size, block_size, x, kv_scale);
249
250 void reshape_and_cache(
264 - torch::Tensor& key, // [num_tokens, num_heads, head_size]
265 - torch::Tensor& value, // [num_tokens, num_heads, head_size]
266 - torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
267 - torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
268 - torch::Tensor& slot_mapping, // [num_tokens]
269 - const std::string& kv_cache_dtype,
270 - const float kv_scale)
271 -{
251 + torch::Tensor& key, // [num_tokens, num_heads, head_size]
252 + torch::Tensor& value, // [num_tokens, num_heads, head_size]
253 + torch::Tensor&
254 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
255 + torch::Tensor&
256 + value_cache, // [num_blocks, num_heads, head_size, block_size]
257 + torch::Tensor& slot_mapping, // [num_tokens]
258 + const std::string& kv_cache_dtype, const float kv_scale) {
259 int num_tokens = key.size(0);
260 int num_heads = key.size(1);
261 int head_size = key.size(2);
@@ -283,17 +270,17 @@ void reshape_and_cache(
270 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
271 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
272
286 - DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype, CALL_RESHAPE_AND_CACHE)
273 + DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype,
274 + CALL_RESHAPE_AND_CACHE)
275 }
276
277 void reshape_and_cache_flash(
290 - torch::Tensor& key, // [num_tokens, num_heads, head_size]
291 - torch::Tensor& value, // [num_tokens, num_heads, head_size]
292 - torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
293 - torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
294 - torch::Tensor& slot_mapping, // [num_tokens]
295 - const std::string& kv_cache_dtype)
296 -{
278 + torch::Tensor& key, // [num_tokens, num_heads, head_size]
279 + torch::Tensor& value, // [num_tokens, num_heads, head_size]
280 + torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
281 + torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
282 + torch::Tensor& slot_mapping, // [num_tokens]
283 + const std::string& kv_cache_dtype) {
284 // FIXME: only support auto datatype, does not support fp8
285 if (kv_cache_dtype != "auto") {
286 TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
@@ -313,62 +300,47 @@ void reshape_and_cache_flash(
300 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
301 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
302 VLLM_DISPATCH_FLOATING_TYPES(
316 - key.scalar_type(),
317 - "reshape_and_cache_flash",
318 - [&] {
319 - vllm::reshape_and_cache_flash_kernel<scalar_t><<<grid, block, 0, stream>>>(
320 - key.data_ptr<scalar_t>(),
321 - value.data_ptr<scalar_t>(),
322 - k_cache.data_ptr<scalar_t>(),
323 - v_cache.data_ptr<scalar_t>(),
324 - slot_mapping.data_ptr<int64_t>(),
325 - block_stride,
326 - key_stride,
327 - value_stride,
328 - num_heads,
329 - head_size,
330 - block_size);
331 - });
303 + key.scalar_type(), "reshape_and_cache_flash", [&] {
304 + vllm::reshape_and_cache_flash_kernel<scalar_t>
305 + <<<grid, block, 0, stream>>>(
306 + key.data_ptr<scalar_t>(), value.data_ptr<scalar_t>(),
307 + k_cache.data_ptr<scalar_t>(), v_cache.data_ptr<scalar_t>(),
308 + slot_mapping.data_ptr<int64_t>(), block_stride, key_stride,
309 + value_stride, num_heads, head_size, block_size);
310 + });
311 }
312
313 namespace vllm {
314
336 -template<typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
337 -__global__ void convert_fp8_kernel(
338 - const Tin* __restrict__ src_cache,
339 - Tout* __restrict__ dst_cache,
340 - const float kv_scale,
341 - const int64_t block_stride) {
315 +template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
316 +__global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
317 + Tout* __restrict__ dst_cache,
318 + const float kv_scale,
319 + const int64_t block_stride) {
320 const int64_t block_idx = blockIdx.x;
321 for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
322 int64_t idx = block_idx * block_stride + i;
345 - dst_cache[idx] = fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
323 + dst_cache[idx] =
324 + fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
325 }
326 }
327
349 -} // namespace vllm
328 +} // namespace vllm
329
351 -#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
352 - vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
353 - reinterpret_cast<Tin*>(src_cache.data_ptr()), \
354 - reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
355 - kv_scale, \
356 - block_stride);
330 +#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
331 + vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
332 + reinterpret_cast<Tin*>(src_cache.data_ptr()), \
333 + reinterpret_cast<Tout*>(dst_cache.data_ptr()), kv_scale, block_stride);
334
335 // Only for testing.
359 -void convert_fp8(
360 - torch::Tensor& dst_cache,
361 - torch::Tensor& src_cache,
362 - const float kv_scale,
363 - const std::string& kv_cache_dtype)
364 -{
336 +void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
337 + const float kv_scale, const std::string& kv_cache_dtype) {
338 torch::Device src_device = src_cache.device();
339 torch::Device dst_device = dst_cache.device();
340 TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
341 TORCH_CHECK(dst_device.is_cuda(), "dst must be on a GPU")
369 - TORCH_CHECK(
370 - src_device.index() == dst_device.index(),
371 - "src and dst must be on the same GPU");
342 + TORCH_CHECK(src_device.index() == dst_device.index(),
343 + "src and dst must be on the same GPU");
344 at::cuda::OptionalCUDAGuard device_guard(src_device);
345
346 int64_t num_blocks = src_cache.size(0);
@@ -398,13 +370,15 @@ void convert_fp8(
370 } else if (src_cache.dtype() == at::ScalarType::Half) {
371 CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
372 } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
401 - CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kFp8E4M3);
373 + CALL_CONVERT_FP8(uint8_t, __nv_bfloat16,
374 + vllm::Fp8KVCacheDataType::kFp8E4M3);
375 } else if (dst_cache.dtype() == at::ScalarType::Float) {
376 CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
377 } else if (dst_cache.dtype() == at::ScalarType::Half) {
378 CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
379 } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
407 - CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
380 + CALL_CONVERT_FP8(__nv_bfloat16, uint8_t,
381 + vllm::Fp8KVCacheDataType::kFp8E4M3);
382 }
383 } else {
384 TORCH_CHECK(false, "Unsupported data type: ", kv_cache_dtype);
csrc/cpu/activation.cpp
+28 -32
@@ -1,10 +1,10 @@
1 #include "cpu_types.hpp"
2
3 namespace {
4 -template <typename scalar_t, vec_op::FP32Vec8 (*func)(const vec_op::FP32Vec8 &),
4 +template <typename scalar_t, vec_op::FP32Vec8 (*func)(const vec_op::FP32Vec8&),
5 bool is_gated>
6 -void activation_kernel(int num_tokens, int d, scalar_t *__restrict__ input,
7 - scalar_t *__restrict__ output) {
6 +void activation_kernel(int num_tokens, int d, scalar_t* __restrict__ input,
7 + scalar_t* __restrict__ output) {
8 using scalar_vec_t = vec_op::vec_t<scalar_t>;
9 constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
10
@@ -34,13 +34,13 @@ void activation_kernel(int num_tokens, int d, scalar_t *__restrict__ input,
34 }
35 }
36
37 -FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8 &x) {
37 +FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8& x) {
38 const vec_op::FP32Vec8 zeros(0.0);
39 const vec_op::FP32Vec8 ones(1.0);
40 return x / (ones + (zeros - x).exp());
41 }
42
43 -FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8 &x) {
43 +FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8& x) {
44 const vec_op::FP32Vec8 ones(1.0);
45 const vec_op::FP32Vec8 w1(0.79788456f);
46 const vec_op::FP32Vec8 w2(0.044715f);
@@ -50,7 +50,7 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8 &x) {
50 return w3 * x * (ones + t);
51 }
52
53 -FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8 &x) {
53 +FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) {
54 const vec_op::FP32Vec8 ones(1.0);
55 const vec_op::FP32Vec8 w1(0.79788456f);
56 const vec_op::FP32Vec8 w2(0.044715f);
@@ -59,14 +59,14 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8 &x) {
59 return w3 * x * (ones + t);
60 }
61
62 -FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8 &x) {
62 +FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) {
63 const vec_op::FP32Vec8 ones(1.0);
64 const vec_op::FP32Vec8 w1(M_SQRT1_2);
65 const vec_op::FP32Vec8 w2(0.5);
66 return x * w2 * (ones + (x * w1).er());
67 }
68
69 -FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8 &x) {
69 +FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8& x) {
70 const vec_op::FP32Vec8 ones(1.0);
71 const vec_op::FP32Vec8 w1(M_SQRT2 * M_2_SQRTPI * 0.5);
72 const vec_op::FP32Vec8 w2(0.5);
@@ -75,40 +75,36 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8 &x) {
75 const vec_op::FP32Vec8 inner = w1 * (x + x_3 * w3);
76 return x * w2 * (ones + inner.tanh());
77 }
78 -}; // namespace
78 +}; // namespace
79
80 -void silu_and_mul(torch::Tensor &out, torch::Tensor &input) {
80 +void silu_and_mul(torch::Tensor& out, torch::Tensor& input) {
81 int num_tokens = input.numel() / input.size(-1);
82 int d = input.size(-1) / 2;
83
84 - VLLM_DISPATCH_FLOATING_TYPES(
85 - input.scalar_type(), "silu_and_mul_impl", [&] {
86 - CPU_KERNEL_GUARD_IN(silu_and_mul_impl)
87 - activation_kernel<scalar_t, silu_act, true>(num_tokens, d,
88 - input.data_ptr<scalar_t>(),
89 - out.data_ptr<scalar_t>());
90 - CPU_KERNEL_GUARD_OUT(silu_and_mul_impl)
91 - });
84 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "silu_and_mul_impl", [&] {
85 + CPU_KERNEL_GUARD_IN(silu_and_mul_impl)
86 + activation_kernel<scalar_t, silu_act, true>(
87 + num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
88 + CPU_KERNEL_GUARD_OUT(silu_and_mul_impl)
89 + });
90 }
91
94 -void gelu_and_mul(torch::Tensor &out, // [..., d]
95 - torch::Tensor &input) // [..., 2 * d]
92 +void gelu_and_mul(torch::Tensor& out, // [..., d]
93 + torch::Tensor& input) // [..., 2 * d]
94 {
95 int num_tokens = input.numel() / input.size(-1);
96 int d = input.size(-1) / 2;
97
100 - VLLM_DISPATCH_FLOATING_TYPES(
101 - input.scalar_type(), "gelu_and_mul_impl", [&] {
102 - CPU_KERNEL_GUARD_IN(gelu_and_mul_impl)
103 - activation_kernel<scalar_t, gelu_act, true>(num_tokens, d,
104 - input.data_ptr<scalar_t>(),
105 - out.data_ptr<scalar_t>());
106 - CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl)
107 - });
98 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul_impl", [&] {
99 + CPU_KERNEL_GUARD_IN(gelu_and_mul_impl)
100 + activation_kernel<scalar_t, gelu_act, true>(
101 + num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
102 + CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl)
103 + });
104 }
105
110 -void gelu_tanh_and_mul(torch::Tensor &out, // [..., d]
111 - torch::Tensor &input) // [..., 2 * d]
106 +void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
107 + torch::Tensor& input) // [..., 2 * d]
108 {
109 int num_tokens = input.numel() / input.size(-1);
110 int d = input.size(-1) / 2;
@@ -123,7 +119,7 @@ void gelu_tanh_and_mul(torch::Tensor &out, // [..., d]
119 });
120 }
121
126 -void gelu_new(torch::Tensor &out, torch::Tensor &input) {
122 +void gelu_new(torch::Tensor& out, torch::Tensor& input) {
123 int num_tokens = input.numel() / input.size(-1);
124 int d = input.size(-1);
125
@@ -135,7 +131,7 @@ void gelu_new(torch::Tensor &out, torch::Tensor &input) {
131 });
132 }
133
138 -void gelu_fast(torch::Tensor &out, torch::Tensor &input) {
134 +void gelu_fast(torch::Tensor& out, torch::Tensor& input) {
135 int num_tokens = input.numel() / input.size(-1);
136 int d = input.size(-1);
137
csrc/cpu/attention.cpp
+205 -206
@@ -2,7 +2,8 @@
2
3 namespace {
4
5 -template <typename scalar_t> struct KernelVecType {
5 +template <typename scalar_t>
6 +struct KernelVecType {
7 using q_load_vec_type = void;
8 using q_vec_type = void;
9 using k_load_vec_type = void;
@@ -11,7 +12,8 @@ template <typename scalar_t> struct KernelVecType {
12 using v_load_vec_type = void;
13 };
14
14 -template <> struct KernelVecType<float> {
15 +template <>
16 +struct KernelVecType<float> {
17 using q_load_vec_type = vec_op::FP32Vec4;
18 using q_vec_type = vec_op::FP32Vec16;
19 using k_load_vec_type = vec_op::FP32Vec16;
@@ -21,7 +23,8 @@ template <> struct KernelVecType<float> {
23 };
24
25 #ifdef __AVX512BF16__
24 -template <> struct KernelVecType<c10::BFloat16> {
26 +template <>
27 +struct KernelVecType<c10::BFloat16> {
28 using q_load_vec_type = vec_op::BF16Vec8;
29 using q_vec_type = vec_op::BF16Vec32;
30 using k_load_vec_type = vec_op::BF16Vec32;
@@ -30,7 +33,8 @@ template <> struct KernelVecType<c10::BFloat16> {
33 using v_load_vec_type = vec_op::BF16Vec16;
34 };
35 #else
33 -template <> struct KernelVecType<c10::BFloat16> {
36 +template <>
37 +struct KernelVecType<c10::BFloat16> {
38 using q_load_vec_type = vec_op::BF16Vec8;
39 using q_vec_type = vec_op::FP32Vec16;
40 using k_load_vec_type = vec_op::BF16Vec16;
@@ -41,7 +45,7 @@ template <> struct KernelVecType<c10::BFloat16> {
45 #endif
46
47 template <typename T>
44 -FORCE_INLINE std::pair<T, T> reduceSoftmax(T *data, const int size,
48 +FORCE_INLINE std::pair<T, T> reduceSoftmax(T* data, const int size,
49 const int capacity) {
50 T max = data[0];
51 for (int i = 1; i < size; ++i) {
@@ -67,10 +71,11 @@ FORCE_INLINE std::pair<T, T> reduceSoftmax(T *data, const int size,
71 }
72
73 template <typename T>
70 -FORCE_INLINE std::pair<T, T>
71 -reduceSoftmaxAlibi(T *data, const int size, const int capacity,
72 - const float alibi_slope, const int start_index,
73 - const int seq_len) {
74 +FORCE_INLINE std::pair<T, T> reduceSoftmaxAlibi(T* data, const int size,
75 + const int capacity,
76 + const float alibi_slope,
77 + const int start_index,
78 + const int seq_len) {
79 data[0] += alibi_slope * (start_index - seq_len + 1);
80 T max = data[0];
81 for (int i = 1; i < size; ++i) {
@@ -98,7 +103,7 @@ reduceSoftmaxAlibi(T *data, const int size, const int capacity,
103 }
104
105 template <typename T>
101 -FORCE_INLINE void reducePartitonSoftmax(const T *max_data, T *sum_data,
106 +FORCE_INLINE void reducePartitonSoftmax(const T* max_data, T* sum_data,
107 const int size) {
108 T max = max_data[0];
109 for (int i = 1; i < size; ++i) {
@@ -132,9 +137,9 @@ struct reduceQKBlockKernel {
137 static_assert(k_load_vec_type::get_elem_num() % x == 0);
138 static_assert(q_load_vec_type::get_elem_num() * sizeof(scalar_t) == 16);
139
135 - FORCE_INLINE static void call(const scalar_t *__restrict__ q,
136 - const scalar_t *__restrict__ k_block,
137 - float *__restrict__ logits, float scale,
140 + FORCE_INLINE static void call(const scalar_t* __restrict__ q,
141 + const scalar_t* __restrict__ k_block,
142 + float* __restrict__ logits, float scale,
143 const int token_num) {
144 const int group_num = (token_num + TOKEN_PER_GROUP - 1) / TOKEN_PER_GROUP;
145
@@ -196,8 +201,8 @@ struct reduceQKBlockKernel {
201
202 template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE,
203 int HEAD_PARTITION_SIZE, typename acc_t>
199 -FORCE_INLINE void reduceValueBlock(const float *prob, const scalar_t *v_block,
200 - acc_t &&acc) {
204 +FORCE_INLINE void reduceValueBlock(const float* prob, const scalar_t* v_block,
205 + acc_t&& acc) {
206 using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
207 constexpr int ELEM_NUM = v_load_vec_type::get_elem_num();
208 static_assert(BLOCK_SIZE == ELEM_NUM);
@@ -209,27 +214,27 @@ FORCE_INLINE void reduceValueBlock(const float *prob, const scalar_t *v_block,
214 acc[head_elem_idx] = acc[head_elem_idx] + prob_vec * fp32_v_vec;
215 });
216 }
212 -}; // namespace
217 +}; // namespace
218
219 // Paged attention v1
220 namespace {
221 template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE>
222 struct paged_attention_v1_impl {
218 - static void
219 - call(scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
220 - const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
221 - const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
223 + static void call(
224 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
225 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
226 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
227 // head_size/x, block_size, x]
223 - const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
228 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
229 // head_size, block_size]
225 - const int num_kv_heads, const float scale,
226 - const int
227 - *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
228 - const int *__restrict__ seq_lens, // [num_seqs]
229 - const int max_num_blocks_per_seq,
230 - const float *__restrict__ alibi_slopes, // [num_heads]
231 - const int q_stride, const int kv_block_stride, const int kv_head_stride,
232 - const int num_seqs, const int num_heads) {
230 + const int num_kv_heads, const float scale,
231 + const int* __restrict__ block_tables, // [num_seqs,
232 + // max_num_blocks_per_seq]
233 + const int* __restrict__ seq_lens, // [num_seqs]
234 + const int max_num_blocks_per_seq,
235 + const float* __restrict__ alibi_slopes, // [num_heads]
236 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
237 + const int num_seqs, const int num_heads) {
238 constexpr int x = 16 / sizeof(scalar_t);
239 const int num_queries_per_kv = num_heads / num_kv_heads;
240
@@ -243,32 +248,31 @@ struct paged_attention_v1_impl {
248
249 size_t logits_bytes =
250 parallel_work_item_num * max_seq_len_padded * sizeof(float);
246 - float *logits = (float *)std::aligned_alloc(
247 - 64, logits_bytes); // Cacheline alignment for each context token.
248 - // [parallel_work_item_num, max_seq_len_padded]
251 + float* logits = (float*)std::aligned_alloc(
252 + 64, logits_bytes); // Cacheline alignment for each context token.
253 + // [parallel_work_item_num, max_seq_len_padded]
254
255 #pragma omp parallel for collapse(2) schedule(dynamic, 1)
256 for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
257 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
258 int seq_len = seq_lens[seq_idx];
254 - const int *seq_block_table =
259 + const int* seq_block_table =
260 block_tables + max_num_blocks_per_seq * seq_idx;
261 const int block_num = (seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
262 const int64_t kv_head_idx = head_idx / num_queries_per_kv;
258 - const scalar_t *__restrict__ q_vec_ptr =
263 + const scalar_t* __restrict__ q_vec_ptr =
264 q + seq_idx * q_stride + head_idx * HEAD_SIZE;
260 - const int last_block_token_num =
261 - seq_len - (block_num - 1) * BLOCK_SIZE;
262 - float *__restrict__ thread_block_logits =
265 + const int last_block_token_num = seq_len - (block_num - 1) * BLOCK_SIZE;
266 + float* __restrict__ thread_block_logits =
267 logits + omp_get_thread_num() * max_seq_len_padded;
268
269 // Compute logits
270 for (int block_idx = 0; block_idx < block_num; ++block_idx) {
271 const int64_t physical_block_idx = seq_block_table[block_idx];
268 - const scalar_t *__restrict__ k_block_cache_ptr =
272 + const scalar_t* __restrict__ k_block_cache_ptr =
273 k_cache + physical_block_idx * kv_block_stride +
274 kv_head_idx * kv_head_stride;
271 - float *__restrict__ head_block_logits =
275 + float* __restrict__ head_block_logits =
276 thread_block_logits + block_idx * BLOCK_SIZE;
277
278 reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
@@ -282,8 +286,7 @@ struct paged_attention_v1_impl {
286 block_num * BLOCK_SIZE, alibi_slopes[head_idx], 0,
287 seq_len);
288 } else {
285 - reduceSoftmax(thread_block_logits, seq_len,
286 - block_num * BLOCK_SIZE);
289 + reduceSoftmax(thread_block_logits, seq_len, block_num * BLOCK_SIZE);
290 }
291
292 // Compute value
@@ -293,14 +296,14 @@ struct paged_attention_v1_impl {
296 for (int head_part_idx = 0; head_part_idx < head_partition_num;
297 ++head_part_idx) {
298 vec_op::FP32Vec16 accums[head_elem_num_per_partition];
296 - scalar_t *__restrict__ out_ptr =
299 + scalar_t* __restrict__ out_ptr =
300 out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
301 head_part_idx * head_elem_num_per_partition;
302 for (int block_idx = 0; block_idx < block_num; ++block_idx) {
303 const int64_t physical_block_idx = seq_block_table[block_idx];
301 - const float *__restrict__ prob_vec_ptr =
304 + const float* __restrict__ prob_vec_ptr =
305 thread_block_logits + block_idx * BLOCK_SIZE;
303 - const scalar_t *__restrict__ v_block_cache_ptr =
306 + const scalar_t* __restrict__ v_block_cache_ptr =
307 v_cache + physical_block_idx * kv_block_stride +
308 kv_head_idx * kv_head_stride +
309 BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -311,7 +314,7 @@ struct paged_attention_v1_impl {
314 if (block_idx != block_num - 1) {
315 const int64_t next_physical_block_idx =
316 seq_block_table[block_idx + 1];
314 - const scalar_t *__restrict__ next_v_block_cache_ptr =
317 + const scalar_t* __restrict__ next_v_block_cache_ptr =
318 v_cache + next_physical_block_idx * kv_block_stride +
319 kv_head_idx * kv_head_stride +
320 BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -340,16 +343,16 @@ struct paged_attention_v1_impl {
343 #define LAUNCH_V1_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
344 paged_attention_v1_impl<T, HEAD_SIZE, BLOCK_SIZE>::call( \
345 out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
343 - block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
346 + block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
347 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, num_seqs, \
348 num_heads);
349
350 template <typename T, int BLOCK_SIZE>
351 void paged_attention_v1_impl_launcher(
349 - torch::Tensor &out, torch::Tensor &query, torch::Tensor &key_cache,
350 - torch::Tensor &value_cache, int num_kv_heads, float scale,
351 - torch::Tensor &block_tables, torch::Tensor &seq_lens,
352 - int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
352 + torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
353 + torch::Tensor& value_cache, int num_kv_heads, float scale,
354 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
355 + const c10::optional<torch::Tensor>& alibi_slopes) {
356 int num_seqs = query.size(0);
357 int num_heads = query.size(1);
358 int head_size = query.size(2);
@@ -359,67 +362,66 @@ void paged_attention_v1_impl_launcher(
362 int kv_head_stride = key_cache.stride(1);
363
364 // NOTE: alibi_slopes is optional.
362 - const float *alibi_slopes_ptr =
365 + const float* alibi_slopes_ptr =
366 alibi_slopes
364 - ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
367 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
368 : nullptr;
369
367 - T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
368 - T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
369 - T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
370 - T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
371 - int *block_tables_ptr = block_tables.data_ptr<int>();
372 - int *seq_lens_ptr = seq_lens.data_ptr<int>();
370 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
371 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
372 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
373 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
374 + int* block_tables_ptr = block_tables.data_ptr<int>();
375 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
376
377 switch (head_size) {
375 - case 64:
376 - LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
377 - break;
378 - case 80:
379 - LAUNCH_V1_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
380 - break;
381 - case 96:
382 - LAUNCH_V1_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
383 - break;
384 - case 112:
385 - LAUNCH_V1_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
386 - break;
387 - case 128:
388 - LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
389 - break;
390 - case 256:
391 - LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
392 - break;
393 - default:
394 - TORCH_CHECK(false, "Unsupported head size: ", head_size);
395 - break;
378 + case 64:
379 + LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
380 + break;
381 + case 80:
382 + LAUNCH_V1_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
383 + break;
384 + case 96:
385 + LAUNCH_V1_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
386 + break;
387 + case 112:
388 + LAUNCH_V1_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
389 + break;
390 + case 128:
391 + LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
392 + break;
393 + case 256:
394 + LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
395 + break;
396 + default:
397 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
398 + break;
399 }
400 }
401
399 -#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
400 - paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
401 - out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
402 +#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
403 + paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
404 + out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
405 seq_lens, max_seq_len, alibi_slopes);
406
404 -#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
405 - switch (block_size) { \
406 - case 16: \
407 - CALL_V1_KERNEL_LAUNCHER(T, 16); \
408 - break; \
409 - default: \
410 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
411 - break; \
407 +#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
408 + switch (block_size) { \
409 + case 16: \
410 + CALL_V1_KERNEL_LAUNCHER(T, 16); \
411 + break; \
412 + default: \
413 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
414 + break; \
415 }
413 -} // namespace
416 +} // namespace
417
415 -void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
416 - torch::Tensor &key_cache, torch::Tensor &value_cache,
418 +void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
419 + torch::Tensor& key_cache, torch::Tensor& value_cache,
420 int num_kv_heads, float scale,
418 - torch::Tensor &block_tables,
419 - torch::Tensor &seq_lens, int block_size,
420 - int max_seq_len,
421 - const c10::optional<torch::Tensor> &alibi_slopes,
422 - const std::string &kv_cache_dtype, float kv_scale) {
421 + torch::Tensor& block_tables, torch::Tensor& seq_lens,
422 + int block_size, int max_seq_len,
423 + const c10::optional<torch::Tensor>& alibi_slopes,
424 + const std::string& kv_cache_dtype, float kv_scale) {
425 TORCH_CHECK(kv_scale == 1.0f);
426 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
427 [&] {
@@ -434,23 +436,24 @@ namespace {
436 template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE, int PARTITION_SIZE>
437 struct paged_attention_v2_impl {
438 static void call(
437 - scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
438 - float *__restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
439 - float
440 - *__restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
441 - scalar_t *__restrict__ tmp_out, // [num_seqs, num_heads,
442 - // max_num_partitions, head_size]
443 - const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
444 - const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
445 - // head_size/x, block_size, x]
446 - const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
447 - // head_size, block_size]
439 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
440 + float* __restrict__ exp_sums, // [num_seqs, num_heads,
441 + // max_num_partitions]
442 + float* __restrict__ max_logits, // [num_seqs, num_heads,
443 + // max_num_partitions]
444 + scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
445 + // max_num_partitions, head_size]
446 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
447 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
448 + // head_size/x, block_size, x]
449 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
450 + // head_size, block_size]
451 const int num_kv_heads, const float scale,
449 - const int
450 - *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
451 - const int *__restrict__ seq_lens, // [num_seqs]
452 + const int* __restrict__ block_tables, // [num_seqs,
453 + // max_num_blocks_per_seq]
454 + const int* __restrict__ seq_lens, // [num_seqs]
455 const int max_num_blocks_per_seq,
453 - const float *__restrict__ alibi_slopes, // [num_heads]
456 + const float* __restrict__ alibi_slopes, // [num_heads]
457 const int q_stride, const int kv_block_stride, const int kv_head_stride,
458 const int num_seqs, const int num_heads, const int max_num_partitions) {
459 constexpr int x = 16 / sizeof(scalar_t);
@@ -468,8 +471,7 @@ struct paged_attention_v2_impl {
471 const int seq_len = seq_lens[seq_idx];
472 const int start_token_idx = partition_idx * PARTITION_SIZE;
473
471 - if (start_token_idx >= seq_len)
472 - continue;
474 + if (start_token_idx >= seq_len) continue;
475
476 const int partition_num =
477 (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
@@ -477,15 +479,14 @@ struct paged_attention_v2_impl {
479 const int token_num =
480 (std::min(seq_len, start_token_idx + PARTITION_SIZE) -
481 start_token_idx);
480 - const int block_num =
481 - (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
482 + const int block_num = (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
483 const int last_block_token_num =
484 token_num - (block_num - 1) * BLOCK_SIZE;
484 - const int *seq_block_table = block_tables +
485 + const int* seq_block_table = block_tables +
486 max_num_blocks_per_seq * seq_idx +
487 start_token_idx / BLOCK_SIZE;
488 const int64_t kv_head_idx = head_idx / num_queries_per_kv;
488 - const scalar_t *__restrict__ q_vec_ptr =
489 + const scalar_t* __restrict__ q_vec_ptr =
490 q + seq_idx * q_stride + head_idx * HEAD_SIZE;
491
492 float logits[PARTITION_SIZE] __attribute__((aligned(64))) = {0};
@@ -493,10 +494,10 @@ struct paged_attention_v2_impl {
494 // Compute logits
495 for (int block_idx = 0; block_idx < block_num; ++block_idx) {
496 const int64_t physical_block_idx = seq_block_table[block_idx];
496 - const scalar_t *__restrict__ k_block_cache_ptr =
497 + const scalar_t* __restrict__ k_block_cache_ptr =
498 k_cache + physical_block_idx * kv_block_stride +
499 kv_head_idx * kv_head_stride;
499 - float *__restrict__ head_block_logits =
500 + float* __restrict__ head_block_logits =
501 logits + block_idx * BLOCK_SIZE;
502
503 reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
@@ -510,13 +511,13 @@ struct paged_attention_v2_impl {
511 logits, token_num, block_num * BLOCK_SIZE,
512 alibi_slopes[head_idx], start_token_idx, seq_len);
513 } else {
513 - max_and_sum = reduceSoftmax(logits, token_num,
514 - block_num * BLOCK_SIZE);
514 + max_and_sum =
515 + reduceSoftmax(logits, token_num, block_num * BLOCK_SIZE);
516 }
517
517 - auto &&[max_logit, exp_sum] = max_and_sum;
518 + auto&& [max_logit, exp_sum] = max_and_sum;
519
519 - scalar_t *__restrict__ output_buffer = nullptr;
520 + scalar_t* __restrict__ output_buffer = nullptr;
521 if (!no_reduce) {
522 auto idx = seq_idx * num_heads * max_num_partitions +
523 head_idx * max_num_partitions + partition_idx;
@@ -538,13 +539,13 @@ struct paged_attention_v2_impl {
539 for (int head_part_idx = 0; head_part_idx < head_partition_num;
540 ++head_part_idx) {
541 vec_op::FP32Vec16 accums[head_elem_num_per_partition];
541 - scalar_t *__restrict__ out_ptr =
542 + scalar_t* __restrict__ out_ptr =
543 output_buffer + head_part_idx * head_elem_num_per_partition;
544 for (int block_idx = 0; block_idx < block_num; ++block_idx) {
545 const int64_t physical_block_idx = seq_block_table[block_idx];
545 - const float *__restrict__ prob_vec_ptr =
546 + const float* __restrict__ prob_vec_ptr =
547 logits + block_idx * BLOCK_SIZE;
547 - const scalar_t *__restrict__ v_block_cache_ptr =
548 + const scalar_t* __restrict__ v_block_cache_ptr =
549 v_cache + physical_block_idx * kv_block_stride +
550 kv_head_idx * kv_head_stride +
551 BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -555,7 +556,7 @@ struct paged_attention_v2_impl {
556 if (block_idx != block_num - 1) {
557 const int64_t next_physical_block_idx =
558 seq_block_table[block_idx + 1];
558 - const scalar_t *__restrict__ next_v_block_cache_ptr =
559 + const scalar_t* __restrict__ next_v_block_cache_ptr =
560 v_cache + next_physical_block_idx * kv_block_stride +
561 kv_head_idx * kv_head_stride +
562 BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -587,8 +588,7 @@ struct paged_attention_v2_impl {
588 const int partition_num =
589 (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
590
590 - if (partition_num == 1)
591 - continue;
591 + if (partition_num == 1) continue;
592
593 reducePartitonSoftmax(
594 max_logits + seq_idx * num_heads * max_num_partitions +
@@ -603,11 +603,11 @@ struct paged_attention_v2_impl {
603 using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
604 static_assert(v_load_vec_type::get_elem_num() == BLOCK_SIZE);
605 constexpr int head_elem_num_per_group =
606 - 16; // Note: didn't align with the cacheline size, due to some HEAD_SIZE
607 - // didn't align with 64 bytes
606 + 16; // Note: didn't align with the cacheline size, due to some
607 + // HEAD_SIZE didn't align with 64 bytes
608 static_assert(HEAD_SIZE % head_elem_num_per_group == 0);
609 constexpr int head_group_num = HEAD_SIZE / head_elem_num_per_group;
610 - const float *__restrict__ rescale_factors = exp_sums;
610 + const float* __restrict__ rescale_factors = exp_sums;
611 #pragma omp parallel for collapse(3) schedule(static, 1)
612 for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
613 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
@@ -616,17 +616,16 @@ struct paged_attention_v2_impl {
616 const int partition_num =
617 (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
618
619 - if (partition_num == 1)
620 - continue;
619 + if (partition_num == 1) continue;
620
622 - const float *__restrict__ seq_head_rescale_factors =
621 + const float* __restrict__ seq_head_rescale_factors =
622 rescale_factors + seq_idx * num_heads * max_num_partitions +
623 head_idx * max_num_partitions;
625 - const scalar_t *__restrict__ seq_head_tmp_out =
624 + const scalar_t* __restrict__ seq_head_tmp_out =
625 tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
626 head_idx * max_num_partitions * HEAD_SIZE +
627 group_idx * head_elem_num_per_group;
629 - scalar_t *__restrict__ seq_head_output =
628 + scalar_t* __restrict__ seq_head_output =
629 out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
630 group_idx * head_elem_num_per_group;
631
@@ -645,21 +644,21 @@ struct paged_attention_v2_impl {
644 }
645 };
646
648 -#define LAUNCH_V2_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
649 - paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
650 - out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
651 - key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
652 - seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
653 - kv_block_stride, kv_head_stride, num_seqs, num_heads, \
647 +#define LAUNCH_V2_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
648 + paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
649 + out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
650 + key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
651 + seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
652 + kv_block_stride, kv_head_stride, num_seqs, num_heads, \
653 max_num_partitions);
654
655 template <typename T, int BLOCK_SIZE, int PARTITION_SIZE = 512>
656 void paged_attention_v2_impl_launcher(
658 - torch::Tensor &out, torch::Tensor &exp_sums, torch::Tensor &max_logits,
659 - torch::Tensor &tmp_out, torch::Tensor &query, torch::Tensor &key_cache,
660 - torch::Tensor &value_cache, int num_kv_heads, float scale,
661 - torch::Tensor &block_tables, torch::Tensor &seq_lens, int block_size,
662 - int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
657 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
658 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
659 + torch::Tensor& value_cache, int num_kv_heads, float scale,
660 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
661 + int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes) {
662 int num_seqs = query.size(0);
663 int num_heads = query.size(1);
664 int head_size = query.size(2);
@@ -670,72 +669,72 @@ void paged_attention_v2_impl_launcher(
669 int max_num_partitions = exp_sums.size(-1);
670
671 // NOTE: alibi_slopes is optional.
673 - const float *alibi_slopes_ptr =
672 + const float* alibi_slopes_ptr =
673 alibi_slopes
675 - ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
674 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
675 : nullptr;
676
678 - T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
679 - float *exp_sums_ptr = reinterpret_cast<float *>(exp_sums.data_ptr());
680 - float *max_logits_ptr = reinterpret_cast<float *>(max_logits.data_ptr());
681 - T *tmp_out_ptr = reinterpret_cast<T *>(tmp_out.data_ptr());
682 - T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
683 - T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
684 - T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
685 - int *block_tables_ptr = block_tables.data_ptr<int>();
686 - int *seq_lens_ptr = seq_lens.data_ptr<int>();
677 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
678 + float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
679 + float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
680 + T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
681 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
682 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
683 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
684 + int* block_tables_ptr = block_tables.data_ptr<int>();
685 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
686
687 switch (head_size) {
689 - case 64:
690 - LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
691 - break;
692 - case 80:
693 - LAUNCH_V2_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
694 - break;
695 - case 96:
696 - LAUNCH_V2_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
697 - break;
698 - case 112:
699 - LAUNCH_V2_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
700 - break;
701 - case 128:
702 - LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
703 - break;
704 - case 256:
705 - LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
706 - break;
707 - default:
708 - TORCH_CHECK(false, "Unsupported head size: ", head_size);
709 - break;
688 + case 64:
689 + LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
690 + break;
691 + case 80:
692 + LAUNCH_V2_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
693 + break;
694 + case 96:
695 + LAUNCH_V2_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
696 + break;
697 + case 112:
698 + LAUNCH_V2_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
699 + break;
700 + case 128:
701 + LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
702 + break;
703 + case 256:
704 + LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
705 + break;
706 + default:
707 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
708 + break;
709 }
710 }
711
713 -#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
714 - paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
715 - out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
716 - num_kv_heads, scale, block_tables, seq_lens, block_size, \
717 - max_seq_len, alibi_slopes);
718 -
719 -#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
720 - switch (block_size) { \
721 - case 16: \
722 - CALL_V2_KERNEL_LAUNCHER(T, 16); \
723 - break; \
724 - default: \
725 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
726 - break; \
712 +#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
713 + paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
714 + out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
715 + num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len, \
716 + alibi_slopes);
717 +
718 +#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
719 + switch (block_size) { \
720 + case 16: \
721 + CALL_V2_KERNEL_LAUNCHER(T, 16); \
722 + break; \
723 + default: \
724 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
725 + break; \
726 }
728 -} // namespace
729 -
730 -void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
731 - torch::Tensor &max_logits, torch::Tensor &tmp_out,
732 - torch::Tensor &query, torch::Tensor &key_cache,
733 - torch::Tensor &value_cache, int num_kv_heads,
734 - float scale, torch::Tensor &block_tables,
735 - torch::Tensor &seq_lens, int block_size,
727 +} // namespace
728 +
729 +void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
730 + torch::Tensor& max_logits, torch::Tensor& tmp_out,
731 + torch::Tensor& query, torch::Tensor& key_cache,
732 + torch::Tensor& value_cache, int num_kv_heads,
733 + float scale, torch::Tensor& block_tables,
734 + torch::Tensor& seq_lens, int block_size,
735 int max_seq_len,
737 - const c10::optional<torch::Tensor> &alibi_slopes,
738 - const std::string &kv_cache_dtype, float kv_scale) {
736 + const c10::optional<torch::Tensor>& alibi_slopes,
737 + const std::string& kv_cache_dtype, float kv_scale) {
738 TORCH_CHECK(kv_scale == 1.0f);
739 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
740 [&] {
csrc/cpu/cache.cpp
+27 -26
@@ -5,25 +5,26 @@
5
6 namespace {
7 template <typename scalar_t>
8 -void copy_blocks_cpu_impl(
9 - std::vector<torch::Tensor> &key_caches,
10 - std::vector<torch::Tensor> &value_caches,
11 - const torch::Tensor& mapping_pairs,
12 - const int element_num_per_block, const int layer_num) {
8 +void copy_blocks_cpu_impl(std::vector<torch::Tensor>& key_caches,
9 + std::vector<torch::Tensor>& value_caches,
10 + const torch::Tensor& mapping_pairs,
11 + const int element_num_per_block,
12 + const int layer_num) {
13 const size_t pair_num = mapping_pairs.size(0);
14 const size_t block_bytes = sizeof(scalar_t) * element_num_per_block;
15 #pragma omp parallel for collapse(2)
16 for (int layer = 0; layer < layer_num; ++layer) {
17 for (size_t pair = 0; pair < pair_num; ++pair) {
18 - int64_t source_offset = element_num_per_block * mapping_pairs[pair][0].item<int64_t>();
18 + int64_t source_offset =
19 + element_num_per_block * mapping_pairs[pair][0].item<int64_t>();
20 int64_t target_offset =
21 element_num_per_block * mapping_pairs[pair][1].item<int64_t>();
21 - scalar_t *key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
22 - scalar_t *source_ptr = key_cache_ptr + source_offset;
23 - scalar_t *target_ptr = key_cache_ptr + target_offset;
22 + scalar_t* key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
23 + scalar_t* source_ptr = key_cache_ptr + source_offset;
24 + scalar_t* target_ptr = key_cache_ptr + target_offset;
25 std::memcpy(target_ptr, source_ptr, block_bytes);
26
26 - scalar_t *value_cache_ptr = value_caches[layer].data_ptr<scalar_t>();
27 + scalar_t* value_cache_ptr = value_caches[layer].data_ptr<scalar_t>();
28 source_ptr = value_cache_ptr + source_offset;
29 target_ptr = value_cache_ptr + target_offset;
30 std::memcpy(target_ptr, source_ptr, block_bytes);
@@ -33,9 +34,9 @@ void copy_blocks_cpu_impl(
34
35 template <typename scalar_t>
36 void reshape_and_cache_cpu_impl(
36 - const scalar_t *__restrict__ key, const scalar_t *__restrict__ value,
37 - scalar_t *__restrict__ key_cache, scalar_t *__restrict__ value_cache,
38 - const int64_t *__restrict__ slot_mapping, const int num_tokens,
37 + const scalar_t* __restrict__ key, const scalar_t* __restrict__ value,
38 + scalar_t* __restrict__ key_cache, scalar_t* __restrict__ value_cache,
39 + const int64_t* __restrict__ slot_mapping, const int num_tokens,
40 const int key_stride, const int value_stride, const int num_heads,
41 const int head_size, const int block_size, const int x) {
42 const int block_elem_num = num_heads * head_size * block_size;
@@ -48,14 +49,14 @@ void reshape_and_cache_cpu_impl(
49 int src_key_head_idx = token_idx * key_stride + head_idx * head_size;
50 int src_value_head_idx =
51 token_idx * value_stride + head_idx * head_size;
51 - const scalar_t *src_key_head_ptr = key + src_key_head_idx;
52 - const scalar_t *src_value_head_ptr = value + src_value_head_idx;
52 + const scalar_t* src_key_head_ptr = key + src_key_head_idx;
53 + const scalar_t* src_value_head_ptr = value + src_value_head_idx;
54 const int64_t block_index = slot_idx / block_size;
55 const int64_t block_offset = slot_idx % block_size;
55 - scalar_t *target_key_head_ptr = key_cache +
56 + scalar_t* target_key_head_ptr = key_cache +
57 block_elem_num * block_index +
58 head_idx * block_size * head_size;
58 - scalar_t *target_value_head_ptr = value_cache +
59 + scalar_t* target_value_head_ptr = value_cache +
60 block_elem_num * block_index +
61 head_idx * block_size * head_size;
62
@@ -79,10 +80,10 @@ void reshape_and_cache_cpu_impl(
80 }
81 }
82 }
82 -}; // namespace
83 +}; // namespace
84
84 -void copy_blocks(std::vector<torch::Tensor> &key_caches,
85 - std::vector<torch::Tensor> &value_caches,
85 +void copy_blocks(std::vector<torch::Tensor>& key_caches,
86 + std::vector<torch::Tensor>& value_caches,
87 const torch::Tensor& block_mapping) {
88 unsigned num_layers = key_caches.size();
89 TORCH_CHECK(num_layers == value_caches.size());
@@ -100,10 +101,10 @@ void copy_blocks(std::vector<torch::Tensor> &key_caches,
101 });
102 }
103
103 -void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
104 - torch::Tensor &key_cache, torch::Tensor &value_cache,
105 - torch::Tensor &slot_mapping,
106 - const std::string &kv_cache_dtype, float kv_scale) {
104 +void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
105 + torch::Tensor& key_cache, torch::Tensor& value_cache,
106 + torch::Tensor& slot_mapping,
107 + const std::string& kv_cache_dtype, float kv_scale) {
108 TORCH_CHECK(kv_scale == 1.0f);
109
110 int num_tokens = key.size(0);
@@ -127,7 +128,7 @@ void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
128 });
129 }
130
130 -void swap_blocks(torch::Tensor &src, torch::Tensor &dst,
131 - const torch::Tensor&block_mapping) {
131 +void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
132 + const torch::Tensor& block_mapping) {
133 TORCH_CHECK(false, "swap_blocks is unsupported on CPU.")
134 }
csrc/cpu/layernorm.cpp
+16 -16
@@ -2,10 +2,10 @@
2
3 namespace {
4 template <typename scalar_t>
5 -void rms_norm_impl(scalar_t *__restrict__ out,
6 - const scalar_t *__restrict__ input,
7 - const scalar_t *__restrict__ weight, const float epsilon,
8 - const int num_tokens, const int hidden_size) {
5 +void rms_norm_impl(scalar_t* __restrict__ out,
6 + const scalar_t* __restrict__ input,
7 + const scalar_t* __restrict__ weight, const float epsilon,
8 + const int num_tokens, const int hidden_size) {
9 using scalar_vec_t = vec_op::vec_t<scalar_t>;
10 constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
11 TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -41,11 +41,11 @@ void rms_norm_impl(scalar_t *__restrict__ out,
41 }
42
43 template <typename scalar_t>
44 -void fused_add_rms_norm_impl(scalar_t *__restrict__ input,
45 - scalar_t *__restrict__ residual,
46 - const scalar_t *__restrict__ weight,
47 - const float epsilon, const int num_tokens,
48 - const int hidden_size) {
44 +void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
45 + scalar_t* __restrict__ residual,
46 + const scalar_t* __restrict__ weight,
47 + const float epsilon, const int num_tokens,
48 + const int hidden_size) {
49 using scalar_vec_t = vec_op::vec_t<scalar_t>;
50 constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
51 TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -85,24 +85,24 @@ void fused_add_rms_norm_impl(scalar_t *__restrict__ input,
85 }
86 }
87 }
88 -} // namespace
88 +} // namespace
89
90 -void rms_norm(torch::Tensor &out, torch::Tensor &input,
91 - torch::Tensor &weight, float epsilon) {
90 +void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
91 + float epsilon) {
92 int hidden_size = input.size(-1);
93 int num_tokens = input.numel() / hidden_size;
94
95 VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_impl", [&] {
96 CPU_KERNEL_GUARD_IN(rms_norm_impl)
97 rms_norm_impl(out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
98 - weight.data_ptr<scalar_t>(), epsilon, num_tokens,
99 - hidden_size);
98 + weight.data_ptr<scalar_t>(), epsilon, num_tokens,
99 + hidden_size);
100 CPU_KERNEL_GUARD_OUT(rms_norm_impl)
101 });
102 }
103
104 -void fused_add_rms_norm(torch::Tensor &input, torch::Tensor &residual,
105 - torch::Tensor &weight, float epsilon) {
104 +void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
105 + torch::Tensor& weight, float epsilon) {
106 int hidden_size = input.size(-1);
107 int num_tokens = input.numel() / hidden_size;
108
csrc/cpu/pos_encoding.cpp
+33 -33
@@ -4,16 +4,16 @@
4 namespace {
5 template <typename scalar_t>
6 void rotary_embedding_impl(
7 - const int64_t
8 - *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
9 - scalar_t
10 - *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
11 - /// [num_tokens, num_heads, head_size]
12 - scalar_t
13 - *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
14 - // [num_tokens, num_kv_heads, head_size]
15 - const scalar_t
16 - *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
7 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or
8 + // [num_tokens]
9 + scalar_t* __restrict__ query, /// [batch_size, seq_len, num_heads,
10 + /// head_size] or [num_tokens, num_heads,
11 + /// head_size]
12 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
13 + // head_size] or [num_tokens, num_kv_heads,
14 + // head_size]
15 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
16 + // 2]
17 const int rot_dim, const int64_t query_stride, const int64_t key_stride,
18 const int num_heads, const int num_kv_heads, const int head_size,
19 const int num_tokens) {
@@ -26,7 +26,7 @@ void rotary_embedding_impl(
26 #pragma omp parallel for
27 for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
28 int64_t pos = positions[token_idx];
29 - const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
29 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
30
31 for (int i = 0; i < num_heads; ++i) {
32 const int head_idx = i;
@@ -94,16 +94,16 @@ void rotary_embedding_impl(
94
95 template <typename scalar_t>
96 void rotary_embedding_gptj_impl(
97 - const int64_t
98 - *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99 - scalar_t
100 - *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
101 - /// [num_tokens, num_heads, head_size]
102 - scalar_t
103 - *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
104 - // [num_tokens, num_kv_heads, head_size]
105 - const scalar_t
106 - *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
97 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or
98 + // [num_tokens]
99 + scalar_t* __restrict__ query, /// [batch_size, seq_len, num_heads,
100 + /// head_size] or [num_tokens, num_heads,
101 + /// head_size]
102 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
103 + // head_size] or [num_tokens, num_kv_heads,
104 + // head_size]
105 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
106 + // 2]
107 const int rot_dim, const int64_t query_stride, const int64_t key_stride,
108 const int num_heads, const int num_kv_heads, const int head_size,
109 const int num_tokens) {
@@ -113,13 +113,13 @@ void rotary_embedding_gptj_impl(
113 for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
114 for (int i = 0; i < num_heads; ++i) {
115 int64_t pos = positions[token_idx];
116 - const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
117 - const scalar_t *cos_cache_ptr = cache_ptr;
118 - const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
116 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
117 + const scalar_t* cos_cache_ptr = cache_ptr;
118 + const scalar_t* sin_cache_ptr = cache_ptr + embed_dim;
119 const int head_idx = i;
120 const int64_t token_head =
121 token_idx * query_stride + head_idx * head_size;
122 - scalar_t *head_query = token_head + query;
122 + scalar_t* head_query = token_head + query;
123 for (int j = 0; j < embed_dim; j += 1) {
124 const int rot_offset = j;
125 const int x_index = 2 * rot_offset;
@@ -141,12 +141,12 @@ void rotary_embedding_gptj_impl(
141 for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
142 for (int i = 0; i < num_kv_heads; ++i) {
143 int64_t pos = positions[token_idx];
144 - const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
145 - const scalar_t *cos_cache_ptr = cache_ptr;
146 - const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
144 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
145 + const scalar_t* cos_cache_ptr = cache_ptr;
146 + const scalar_t* sin_cache_ptr = cache_ptr + embed_dim;
147 const int head_idx = i;
148 const int64_t token_head = token_idx * key_stride + head_idx * head_size;
149 - scalar_t *head_key = key + token_head;
149 + scalar_t* head_key = key + token_head;
150 for (int j = 0; j < embed_dim; j += 1) {
151 const int rot_offset = j;
152 const int x_index = 2 * rot_offset;
@@ -164,11 +164,11 @@ void rotary_embedding_gptj_impl(
164 }
165 }
166 }
167 -}; // namespace
167 +}; // namespace
168
169 -void rotary_embedding(torch::Tensor &positions, torch::Tensor &query,
170 - torch::Tensor &key, int head_size,
171 - torch::Tensor &cos_sin_cache, bool is_neox) {
169 +void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
170 + torch::Tensor& key, int head_size,
171 + torch::Tensor& cos_sin_cache, bool is_neox) {
172 int num_tokens = query.numel() / query.size(-1);
173 int rot_dim = cos_sin_cache.size(1);
174 int num_heads = query.size(-1) / head_size;
csrc/cpu/pybind.cpp
+23 -52
@@ -8,66 +8,37 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8 pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
10 // Attention ops
11 - ops.def(
12 - "paged_attention_v1",
13 - &paged_attention_v1,
14 - "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15 - ops.def(
16 - "paged_attention_v2",
17 - &paged_attention_v2,
18 - "PagedAttention V2.");
11 + ops.def("paged_attention_v1", &paged_attention_v1,
12 + "Compute the attention between an input query and the cached "
13 + "keys/values using PagedAttention.");
14 + ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15
16 // Activation ops
21 - ops.def(
22 - "silu_and_mul",
23 - &silu_and_mul,
24 - "Activation function used in SwiGLU.");
25 - ops.def(
26 - "gelu_and_mul",
27 - &gelu_and_mul,
28 - "Activation function used in GeGLU with `none` approximation.");
29 - ops.def(
30 - "gelu_tanh_and_mul",
31 - &gelu_tanh_and_mul,
32 - "Activation function used in GeGLU with `tanh` approximation.");
33 - ops.def(
34 - "gelu_new",
35 - &gelu_new,
36 - "GELU implementation used in GPT-2.");
37 - ops.def(
38 - "gelu_fast",
39 - &gelu_fast,
40 - "Approximate GELU implementation.");
17 + ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18 + ops.def("gelu_and_mul", &gelu_and_mul,
19 + "Activation function used in GeGLU with `none` approximation.");
20 + ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21 + "Activation function used in GeGLU with `tanh` approximation.");
22 + ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23 + ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24
25 // Layernorm
43 - ops.def(
44 - "rms_norm",
45 - &rms_norm,
46 - "Apply Root Mean Square (RMS) Normalization to the input tensor.");
26 + ops.def("rms_norm", &rms_norm,
27 + "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28
48 - ops.def(
49 - "fused_add_rms_norm",
50 - &fused_add_rms_norm,
51 - "In-place fused Add and RMS Normalization");
29 + ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30 + "In-place fused Add and RMS Normalization");
31
32 // Rotary embedding
54 - ops.def(
55 - "rotary_embedding",
56 - &rotary_embedding,
57 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
33 + ops.def("rotary_embedding", &rotary_embedding,
34 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35
36 // Cache ops
37 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
61 - cache_ops.def(
62 - "swap_blocks",
63 - &swap_blocks,
64 - "Swap in (out) the cache blocks from src to dst");
65 - cache_ops.def(
66 - "copy_blocks",
67 - &copy_blocks,
68 - "Copy the cache blocks from src to dst");
69 - cache_ops.def(
70 - "reshape_and_cache",
71 - &reshape_and_cache,
72 - "Reshape the key and value tensors and cache them");
38 + cache_ops.def("swap_blocks", &swap_blocks,
39 + "Swap in (out) the cache blocks from src to dst");
40 + cache_ops.def("copy_blocks", &copy_blocks,
41 + "Copy the cache blocks from src to dst");
42 + cache_ops.def("reshape_and_cache", &reshape_and_cache,
43 + "Reshape the key and value tensors and cache them");
44 }
csrc/cuda_compat.h
+5 -4
@@ -1,7 +1,7 @@
1 #pragma once
2
3 #ifdef USE_ROCM
4 -#include <hip/hip_runtime.h>
4 + #include <hip/hip_runtime.h>
5 #endif
6
7 #ifndef USE_ROCM
@@ -17,7 +17,8 @@
17 #endif
18
19 #ifndef USE_ROCM
20 - #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor_sync(uint32_t(-1), var, lane_mask)
20 + #define VLLM_SHFL_XOR_SYNC(var, lane_mask) \
21 + __shfl_xor_sync(uint32_t(-1), var, lane_mask)
22 #else
23 #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
24 #endif
@@ -29,7 +30,8 @@
30 #endif
31
32 #ifndef USE_ROCM
32 - #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down_sync(uint32_t(-1), var, lane_delta)
33 + #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) \
34 + __shfl_down_sync(uint32_t(-1), var, lane_delta)
35 #else
36 #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down(var, lane_delta)
37 #endif
@@ -41,4 +43,3 @@
43 #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
44 hipFuncSetAttribute(FUNC, hipFuncAttributeMaxDynamicSharedMemorySize, VAL)
45 #endif
44 -
csrc/cuda_utils.h
+2 -5
@@ -2,9 +2,6 @@
2
3 #include <torch/extension.h>
4
5 -int get_device_attribute(
6 - int attribute,
7 - int device_id);
5 +int get_device_attribute(int attribute, int device_id);
6
9 -int get_max_shared_memory_per_block_device_attribute(
10 - int device_id);
7 +int get_max_shared_memory_per_block_device_attribute(int device_id);
csrc/cuda_utils_kernels.cu
+17 -23
@@ -2,34 +2,28 @@
2 #include <hip/hip_runtime.h>
3 #include <hip/hip_runtime_api.h>
4 #endif
5 -int get_device_attribute(
6 - int attribute,
7 - int device_id)
8 -{
9 - int device, value;
10 - if (device_id < 0) {
11 - cudaGetDevice(&device);
12 - }
13 - else {
14 - device = device_id;
15 - }
16 - cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute), device);
17 - return value;
5 +int get_device_attribute(int attribute, int device_id) {
6 + int device, value;
7 + if (device_id < 0) {
8 + cudaGetDevice(&device);
9 + } else {
10 + device = device_id;
11 + }
12 + cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute),
13 + device);
14 + return value;
15 }
16
20 -
21 -int get_max_shared_memory_per_block_device_attribute(
22 - int device_id)
23 -{
24 -int attribute;
25 -// https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
26 -// cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
17 +int get_max_shared_memory_per_block_device_attribute(int device_id) {
18 + int attribute;
19 + // https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
20 + // cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
21
22 #ifdef USE_ROCM
29 - attribute = hipDeviceAttributeMaxSharedMemoryPerBlock;
23 + attribute = hipDeviceAttributeMaxSharedMemoryPerBlock;
24 #else
31 - attribute = cudaDevAttrMaxSharedMemoryPerBlockOptin;
25 + attribute = cudaDevAttrMaxSharedMemoryPerBlockOptin;
26 #endif
27
34 - return get_device_attribute(attribute, device_id);
28 + return get_device_attribute(attribute, device_id);
29 }
csrc/custom_all_reduce.cu
+27 -28
@@ -7,11 +7,11 @@
7
8 // fake pointer type
9 using fptr_t = uint64_t;
10 -static_assert(sizeof(void *) == sizeof(fptr_t));
10 +static_assert(sizeof(void*) == sizeof(fptr_t));
11
12 -fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
13 - const std::vector<std::string> &handles,
14 - const std::vector<int64_t> &offsets, int rank,
12 +fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13 + const std::vector<std::string>& handles,
14 + const std::vector<int64_t>& offsets, int rank,
15 bool full_nvlink) {
16 int world_size = offsets.size();
17 if (world_size > 8)
@@ -29,7 +29,7 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
29 std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30 }
31 return (fptr_t) new vllm::CustomAllreduce(
32 - reinterpret_cast<vllm::Signal *>(meta.data_ptr()), rank_data.data_ptr(),
32 + reinterpret_cast<vllm::Signal*>(meta.data_ptr()), rank_data.data_ptr(),
33 rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34 }
35
@@ -49,13 +49,13 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
49 * 5. A[None].expand(2, -1, -1, -1): Not OK
50 * 6. A[:, 1:, 1:]: Not OK
51 */
52 -bool _is_weak_contiguous(torch::Tensor &t) {
52 +bool _is_weak_contiguous(torch::Tensor& t) {
53 return t.is_contiguous() ||
54 (t.storage().nbytes() - t.storage_offset() * t.element_size() ==
55 t.numel() * t.element_size());
56 }
57
58 -bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
58 +bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
59 bool full_nvlink) {
60 auto inp_size = inp.numel() * inp.element_size();
61 // custom allreduce requires input byte size to be multiples of 16
@@ -67,28 +67,27 @@ bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
67 return false;
68 }
69
70 -void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
70 +void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
71 cudaStream_t stream) {
72 - auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
72 + auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
73 TORCH_CHECK(_is_weak_contiguous(out));
74 switch (out.scalar_type()) {
75 case at::ScalarType::Float: {
76 - fa->allreduce<float>(stream, reinterpret_cast<float *>(inp.data_ptr()),
77 - reinterpret_cast<float *>(out.data_ptr()),
76 + fa->allreduce<float>(stream, reinterpret_cast<float*>(inp.data_ptr()),
77 + reinterpret_cast<float*>(out.data_ptr()),
78 out.numel());
79 break;
80 }
81 case at::ScalarType::Half: {
82 - fa->allreduce<half>(stream, reinterpret_cast<half *>(inp.data_ptr()),
83 - reinterpret_cast<half *>(out.data_ptr()),
84 - out.numel());
82 + fa->allreduce<half>(stream, reinterpret_cast<half*>(inp.data_ptr()),
83 + reinterpret_cast<half*>(out.data_ptr()), out.numel());
84 break;
85 }
86 #if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
87 case at::ScalarType::BFloat16: {
88 fa->allreduce<nv_bfloat16>(
90 - stream, reinterpret_cast<nv_bfloat16 *>(inp.data_ptr()),
91 - reinterpret_cast<nv_bfloat16 *>(out.data_ptr()), out.numel());
89 + stream, reinterpret_cast<nv_bfloat16*>(inp.data_ptr()),
90 + reinterpret_cast<nv_bfloat16*>(out.data_ptr()), out.numel());
91 break;
92 }
93 #endif
@@ -98,7 +97,7 @@ void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
97 }
98 }
99
101 -void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
100 +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) {
101 const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
102 auto stream = c10::cuda::getCurrentCUDAStream().stream();
103 TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
@@ -106,8 +105,8 @@ void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
105 _all_reduce(_fa, inp, out, stream);
106 }
107
109 -void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &reg_buffer,
110 - torch::Tensor &out) {
108 +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
109 + torch::Tensor& out) {
110 const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
111 auto stream = c10::cuda::getCurrentCUDAStream().stream();
112
@@ -122,27 +121,27 @@ void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &reg_buffer,
121 }
122
123 void dispose(fptr_t _fa) {
125 - auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
124 + auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
125 delete fa;
126 }
127
128 int meta_size() { return sizeof(vllm::Signal); }
129
131 -void register_buffer(fptr_t _fa, torch::Tensor &t,
132 - const std::vector<std::string> &handles,
133 - const std::vector<int64_t> &offsets) {
134 - auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
130 +void register_buffer(fptr_t _fa, torch::Tensor& t,
131 + const std::vector<std::string>& handles,
132 + const std::vector<int64_t>& offsets) {
133 + auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
134 fa->register_buffer(handles, offsets, t.data_ptr());
135 }
136
137 std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
138 fptr_t _fa) {
140 - auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
139 + auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140 return fa->get_graph_buffer_ipc_meta();
141 }
142
144 -void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
145 - const std::vector<std::vector<int64_t>> &offsets) {
146 - auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
143 +void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
144 + const std::vector<std::vector<int64_t>>& offsets) {
145 + auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
146 fa->register_graph_buffers(handles, offsets);
147 }
csrc/custom_all_reduce.cuh
+51 -54
@@ -31,9 +31,9 @@ struct Signal {
31 alignas(128) uint32_t end[kMaxBlocks][8];
32 };
33
34 -struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
34 +struct __align__(16) RankData { const void* __restrict__ ptrs[8]; };
35
36 -struct __align__(16) RankSignals { volatile Signal *signals[8]; };
36 +struct __align__(16) RankSignals { volatile Signal* signals[8]; };
37
38 // like std::array, but aligned
39 template <typename T, int sz>
@@ -68,11 +68,11 @@ DINLINE half downcast_s(float val) {
68 // scalar add functions
69 // for some reason when compiling with Pytorch, the + operator for half and
70 // bfloat is disabled so we call the intrinsics directly
71 -DINLINE half &assign_add(half &a, half b) {
71 +DINLINE half& assign_add(half& a, half b) {
72 a = __hadd(a, b);
73 return a;
74 }
75 -DINLINE float &assign_add(float &a, float b) { return a += b; }
75 +DINLINE float& assign_add(float& a, float b) { return a += b; }
76
77 #if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
78 DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
@@ -80,14 +80,14 @@ template <>
80 DINLINE nv_bfloat16 downcast_s(float val) {
81 return __float2bfloat16(val);
82 }
83 -DINLINE nv_bfloat16 &assign_add(nv_bfloat16 &a, nv_bfloat16 b) {
83 +DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) {
84 a = __hadd(a, b);
85 return a;
86 }
87 #endif
88
89 template <typename T, int N>
90 -DINLINE array_t<T, N> &packed_assign_add(array_t<T, N> &a, array_t<T, N> b) {
90 +DINLINE array_t<T, N>& packed_assign_add(array_t<T, N>& a, array_t<T, N> b) {
91 #pragma unroll
92 for (int i = 0; i < N; i++) {
93 assign_add(a.data[i], b.data[i]);
@@ -128,7 +128,7 @@ DINLINE O downcast(array_t<float, O::size> val) {
128 // prior memory accesses. Note: volatile writes will not be reordered against
129 // other volatile writes.
130 template <int ngpus>
131 -DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
131 +DINLINE void start_sync(const RankSignals& sg, volatile Signal* self_sg,
132 int rank) {
133 if (threadIdx.x < ngpus) {
134 // reset flag for next time
@@ -137,8 +137,7 @@ DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
137 // Latency = 1 p2p write
138 sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1;
139 // wait until we got true from all ranks
140 - while (!self_sg->start[blockIdx.x][threadIdx.x])
141 - ;
140 + while (!self_sg->start[blockIdx.x][threadIdx.x]);
141 }
142 __syncthreads();
143 }
@@ -147,13 +146,13 @@ DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
146 // barrier in the all reduce kernel. If it's the final synchronization barrier,
147 // we don't need to make any visibility guarantees for prior memory accesses.
148 template <int ngpus, bool final_sync = false>
150 -DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
149 +DINLINE void end_sync(const RankSignals& sg, volatile Signal* self_sg,
150 int rank) {
151 __syncthreads();
152 // eliminate the case that prior writes are not visible after signals become
153 // visible. Note that I did not managed to make this happen through a lot of
154 // testing. Might be the case that hardware provides stronger guarantee than
156 - // the memory model.
155 + // the memory model.
156 if constexpr (!final_sync) __threadfence_system();
157 if (threadIdx.x < ngpus) {
158 // reset flag for next time
@@ -162,14 +161,13 @@ DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
161 // Latency = 1 p2p write
162 sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1;
163 // wait until we got true from all ranks
165 - while (!self_sg->end[blockIdx.x][threadIdx.x])
166 - ;
164 + while (!self_sg->end[blockIdx.x][threadIdx.x]);
165 }
166 if constexpr (!final_sync) __syncthreads();
167 }
168
169 template <typename P, int ngpus, typename A>
172 -DINLINE P packed_reduce(const P *ptrs[], int idx) {
170 +DINLINE P packed_reduce(const P* ptrs[], int idx) {
171 A tmp = upcast(ptrs[0][idx]);
172 #pragma unroll
173 for (int i = 1; i < ngpus; i++) {
@@ -180,8 +178,8 @@ DINLINE P packed_reduce(const P *ptrs[], int idx) {
178
179 template <typename T, int ngpus>
180 __global__ void __launch_bounds__(512, 1)
183 - cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
184 - volatile Signal *self_sg, T *__restrict__ result,
181 + cross_device_reduce_1stage(RankData* _dp, RankSignals sg,
182 + volatile Signal* self_sg, T* __restrict__ result,
183 int rank, int size) {
184 using P = typename packed_t<T>::P;
185 using A = typename packed_t<T>::A;
@@ -192,21 +190,20 @@ __global__ void __launch_bounds__(512, 1)
190 // do the actual reduction
191 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
192 idx += gridDim.x * blockDim.x) {
195 - ((P *)result)[idx] =
196 - packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
193 + ((P*)result)[idx] = packed_reduce<P, ngpus, A>((const P**)&dp.ptrs[0], idx);
194 }
195 end_sync<ngpus, true>(sg, self_sg, rank);
196 }
197
198 template <typename P>
202 -DINLINE P *get_tmp_buf(volatile Signal *sg) {
203 - return (P *)(((Signal *)sg) + 1);
199 +DINLINE P* get_tmp_buf(volatile Signal* sg) {
200 + return (P*)(((Signal*)sg) + 1);
201 }
202
203 template <typename T, int ngpus>
204 __global__ void __launch_bounds__(512, 1)
208 - cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
209 - volatile Signal *self_sg, T *__restrict__ result,
205 + cross_device_reduce_2stage(RankData* _dp, RankSignals sg,
206 + volatile Signal* self_sg, T* __restrict__ result,
207 int rank, int size) {
208 int tid = blockIdx.x * blockDim.x + threadIdx.x;
209 int stride = gridDim.x * blockDim.x;
@@ -216,12 +213,12 @@ __global__ void __launch_bounds__(512, 1)
213 int start = rank * part;
214 int end = rank == ngpus - 1 ? size : start + part;
215 int largest_part = part + size % ngpus;
219 - const P *ptrs[ngpus];
220 - P *tmps[ngpus];
216 + const P* ptrs[ngpus];
217 + P* tmps[ngpus];
218 #pragma unroll
219 for (int i = 0; i < ngpus; i++) {
220 int target = (rank + i) % ngpus;
224 - ptrs[i] = (const P *)_dp->ptrs[target];
221 + ptrs[i] = (const P*)_dp->ptrs[target];
222 tmps[i] = get_tmp_buf<P>(sg.signals[target]);
223 }
224 auto tmp_out = tmps[0];
@@ -243,7 +240,7 @@ __global__ void __launch_bounds__(512, 1)
240 int gather_from_rank = ((rank + i) % ngpus);
241 if (gather_from_rank == ngpus - 1 || idx < part) {
242 int dst_idx = gather_from_rank * part + idx;
246 - ((P *)result)[dst_idx] = tmps[i][idx];
243 + ((P*)result)[dst_idx] = tmps[i][idx];
244 }
245 }
246 }
@@ -261,14 +258,14 @@ class CustomAllreduce {
258
259 // below are device pointers
260 RankSignals sg_;
264 - std::unordered_map<void *, RankData *> buffers_;
265 - Signal *self_sg_;
261 + std::unordered_map<void*, RankData*> buffers_;
262 + Signal* self_sg_;
263
264 // stores the registered device pointers from all ranks
265 RankData *d_rank_data_base_, *d_rank_data_end_;
269 - std::vector<void *> graph_unreg_buffers_;
266 + std::vector<void*> graph_unreg_buffers_;
267 // a map from IPC handles to opened IPC pointers
271 - std::map<IPC_KEY, char *> ipc_handles_;
268 + std::map<IPC_KEY, char*> ipc_handles_;
269
270 /**
271 * meta is a pointer to device metadata and temporary buffer for allreduce.
@@ -279,22 +276,22 @@ class CustomAllreduce {
276 * note: this class does not own any device memory. Any required buffers
277 * are passed in from the constructor
278 */
282 - CustomAllreduce(Signal *meta, void *rank_data, size_t rank_data_sz,
283 - const cudaIpcMemHandle_t *handles,
284 - const std::vector<int64_t> &offsets, int rank,
279 + CustomAllreduce(Signal* meta, void* rank_data, size_t rank_data_sz,
280 + const cudaIpcMemHandle_t* handles,
281 + const std::vector<int64_t>& offsets, int rank,
282 bool full_nvlink = true)
283 : rank_(rank),
284 world_size_(offsets.size()),
285 full_nvlink_(full_nvlink),
286 self_sg_(meta),
290 - d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
287 + d_rank_data_base_(reinterpret_cast<RankData*>(rank_data)),
288 d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
289 for (int i = 0; i < world_size_; i++) {
293 - Signal *rank_sg;
290 + Signal* rank_sg;
291 if (i != rank_) {
295 - char *handle = open_ipc_handle(&handles[i]);
292 + char* handle = open_ipc_handle(&handles[i]);
293 handle += offsets[i];
297 - rank_sg = (Signal *)handle;
294 + rank_sg = (Signal*)handle;
295 } else {
296 rank_sg = self_sg_;
297 }
@@ -302,13 +299,13 @@ class CustomAllreduce {
299 }
300 }
301
305 - char *open_ipc_handle(const void *ipc_handle) {
302 + char* open_ipc_handle(const void* ipc_handle) {
303 auto [it, new_handle] =
307 - ipc_handles_.insert({*((IPC_KEY *)ipc_handle), nullptr});
304 + ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr});
305 if (new_handle) {
309 - char *ipc_ptr;
310 - CUDACHECK(cudaIpcOpenMemHandle((void **)&ipc_ptr,
311 - *((const cudaIpcMemHandle_t *)ipc_handle),
306 + char* ipc_ptr;
307 + CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptr,
308 + *((const cudaIpcMemHandle_t*)ipc_handle),
309 cudaIpcMemLazyEnablePeerAccess));
310 it->second = ipc_ptr;
311 }
@@ -323,7 +320,7 @@ class CustomAllreduce {
320 std::vector<int64_t> offsets(num_buffers);
321 for (int i = 0; i < num_buffers; i++) {
322 auto ptr = graph_unreg_buffers_[i];
326 - void *base_ptr;
323 + void* base_ptr;
324 // note: must share the base address of each allocation, or we get wrong
325 // address
326 if (cuPointerGetAttribute(&base_ptr,
@@ -331,8 +328,8 @@ class CustomAllreduce {
328 (CUdeviceptr)ptr) != CUDA_SUCCESS)
329 throw std::runtime_error("failed to get pointer attr");
330 CUDACHECK(cudaIpcGetMemHandle(
334 - (cudaIpcMemHandle_t *)&handles[i * handle_sz], base_ptr));
335 - offsets[i] = ((char *)ptr) - ((char *)base_ptr);
331 + (cudaIpcMemHandle_t*)&handles[i * handle_sz], base_ptr));
332 + offsets[i] = ((char*)ptr) - ((char*)base_ptr);
333 }
334 return std::make_pair(handles, offsets);
335 }
@@ -344,13 +341,13 @@ class CustomAllreduce {
341 std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
342 }
343
347 - void register_buffer(const std::vector<std::string> &handles,
348 - const std::vector<int64_t> &offsets, void *self) {
344 + void register_buffer(const std::vector<std::string>& handles,
345 + const std::vector<int64_t>& offsets, void* self) {
346 check_rank_data_capacity();
347 RankData data;
348 for (int i = 0; i < world_size_; i++) {
349 if (i != rank_) {
353 - char *handle = open_ipc_handle(handles[i].data());
350 + char* handle = open_ipc_handle(handles[i].data());
351 handle += offsets[i];
352 data.ptrs[i] = handle;
353 } else {
@@ -371,17 +368,17 @@ class CustomAllreduce {
368 // got a different address. IPC handles have internal reference counting
369 // mechanism so overhead should be small.
370 void register_graph_buffers(
374 - const std::vector<std::string> &handles,
375 - const std::vector<std::vector<int64_t>> &offsets) {
371 + const std::vector<std::string>& handles,
372 + const std::vector<std::vector<int64_t>>& offsets) {
373 auto num_buffers = graph_unreg_buffers_.size();
374 check_rank_data_capacity(num_buffers);
375 std::vector<RankData> rank_data(num_buffers);
376 for (int i = 0; i < num_buffers; i++) {
377 auto self_ptr = graph_unreg_buffers_[i];
381 - auto &rd = rank_data[i];
378 + auto& rd = rank_data[i];
379 for (int j = 0; j < world_size_; j++) {
380 if (j != rank_) {
384 - char *handle =
381 + char* handle =
382 open_ipc_handle(&handles[j][i * sizeof(cudaIpcMemHandle_t)]);
383 handle += offsets[j][i];
384 rd.ptrs[j] = handle;
@@ -405,7 +402,7 @@ class CustomAllreduce {
402 * will cause contention on NVLink bus.
403 */
404 template <typename T>
408 - void allreduce(cudaStream_t stream, T *input, T *output, int size,
405 + void allreduce(cudaStream_t stream, T* input, T* output, int size,
406 int threads = 512, int block_limit = 36) {
407 auto d = packed_t<T>::P::size;
408 if (size % d != 0)
@@ -418,7 +415,7 @@ class CustomAllreduce {
415 std::to_string(kMaxBlocks) + ". Got " +
416 std::to_string(block_limit));
417
421 - RankData *ptrs;
418 + RankData* ptrs;
419 cudaStreamCaptureStatus status;
420 CUDACHECK(cudaStreamIsCapturing(stream, &status));
421 if (status == cudaStreamCaptureStatusActive) {
csrc/custom_all_reduce_test.cu
+19 -19
@@ -48,7 +48,7 @@ __global__ void dummy_kernel() {
48 }
49
50 template <typename T>
51 -__global__ void set_data(T *data, int size, int myRank) {
51 +__global__ void set_data(T* data, int size, int myRank) {
52 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
53 idx += gridDim.x * blockDim.x) {
54 data[idx] = myRank * 0.11f;
@@ -56,8 +56,8 @@ __global__ void set_data(T *data, int size, int myRank) {
56 }
57
58 template <typename T>
59 -__global__ void convert_data(const T *data1, const T *data2, double *fdata1,
60 - double *fdata2, int size) {
59 +__global__ void convert_data(const T* data1, const T* data2, double* fdata1,
60 + double* fdata2, int size) {
61 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
62 idx += gridDim.x * blockDim.x) {
63 fdata1[idx] = data1[idx];
@@ -65,7 +65,7 @@ __global__ void convert_data(const T *data1, const T *data2, double *fdata1,
65 }
66 }
67
68 -__global__ void init_rand(curandState_t *state, int size, int nRanks) {
68 +__global__ void init_rand(curandState_t* state, int size, int nRanks) {
69 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
70 idx += gridDim.x * blockDim.x) {
71 for (int i = 0; i < nRanks; i++) {
@@ -75,7 +75,7 @@ __global__ void init_rand(curandState_t *state, int size, int nRanks) {
75 }
76
77 template <typename T>
78 -__global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
78 +__global__ void gen_data(curandState_t* state, T* data, double* ground_truth,
79 int myRank, int nRanks, int size) {
80 for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
81 idx += gridDim.x * blockDim.x) {
@@ -91,9 +91,9 @@ __global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
91 }
92
93 template <typename T>
94 -void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
94 +void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
95 int data_size, bool performance_test) {
96 - T *result;
96 + T* result;
97 cudaStream_t stream;
98 CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
99 CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
@@ -101,8 +101,8 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
101
102 cudaIpcMemHandle_t self_data_handle;
103 cudaIpcMemHandle_t data_handles[8];
104 - vllm::Signal *buffer;
105 - T *self_data_copy;
104 + vllm::Signal* buffer;
105 + T* self_data_copy;
106 /**
107 * Allocate IPC buffer
108 *
@@ -125,22 +125,22 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
125 MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
126 MPI_BYTE, MPI_COMM_WORLD));
127
128 - void *rank_data;
128 + void* rank_data;
129 size_t rank_data_sz = 16 * 1024 * 1024;
130 CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
131 std::vector<int64_t> offsets(nRanks, 0);
132 vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
133 offsets, myRank);
134 - auto *self_data =
135 - reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136 - sizeof(vllm::Signal) + data_size * sizeof(T));
134 + auto* self_data =
135 + reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
136 + sizeof(vllm::Signal) + data_size * sizeof(T));
137 // hack buffer registration
138 {
139 std::vector<std::string> handles;
140 handles.reserve(nRanks);
141 for (int i = 0; i < nRanks; i++) {
142 - char *begin = (char *)&data_handles[i];
143 - char *end = (char *)&data_handles[i + 1];
142 + char* begin = (char*)&data_handles[i];
143 + char* end = (char*)&data_handles[i + 1];
144 handles.emplace_back(begin, end);
145 }
146 std::vector<int64_t> offsets(nRanks,
@@ -148,9 +148,9 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
148 fa.register_buffer(handles, offsets, self_data);
149 }
150
151 - double *ground_truth;
151 + double* ground_truth;
152 CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
153 - curandState_t *states;
153 + curandState_t* states;
154 CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
155 init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
156 gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
@@ -287,7 +287,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
287 CUDACHECK(cudaStreamDestroy(stream));
288 }
289
290 -int main(int argc, char **argv) {
290 +int main(int argc, char** argv) {
291 int nRanks, myRank;
292 MPICHECK(MPI_Init(&argc, &argv));
293 MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
@@ -296,7 +296,7 @@ int main(int argc, char **argv) {
296 ncclUniqueId id;
297 ncclComm_t comm;
298 if (myRank == 0) ncclGetUniqueId(&id);
299 - MPICHECK(MPI_Bcast(static_cast<void *>(&id), sizeof(id), MPI_BYTE, 0,
299 + MPICHECK(MPI_Bcast(static_cast<void*>(&id), sizeof(id), MPI_BYTE, 0,
300 MPI_COMM_WORLD));
301 NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
302
csrc/dispatch_utils.h
+20 -22
@@ -6,32 +6,30 @@
6
7 #include <torch/extension.h>
8
9 -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
11 - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
9 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
11 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
12 AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
13
14 -#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15 - AT_DISPATCH_SWITCH( \
16 - TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
14 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
16
18 -#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
19 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
20 - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
21 - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
17 +#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
18 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
19 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
20 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
21 AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
22
24 -#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25 - AT_DISPATCH_SWITCH( \
26 - TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
27 -
28 -#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
29 - AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
30 - AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
31 - AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
32 - AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
23 +#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
24 + AT_DISPATCH_SWITCH(TYPE, NAME, \
25 + VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
26 +
27 +#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
28 + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
29 + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
30 + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
31 + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
32 AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__)
33
35 -#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
36 - AT_DISPATCH_SWITCH( \
37 - TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
34 +#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
35 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
csrc/layernorm_kernels.cu
+121 -121
@@ -11,26 +11,24 @@
11 #include <hip/hip_bf16.h>
12 #include <hip/hip_fp16.h>
13
14 - using __nv_bfloat16 = __hip_bfloat16;
15 - using __nv_bfloat162 = __hip_bfloat162;
14 +using __nv_bfloat16 = __hip_bfloat16;
15 +using __nv_bfloat162 = __hip_bfloat162;
16 #endif
17
18 namespace vllm {
19
20 // TODO(woosuk): Further optimize this kernel.
21 -template<typename scalar_t>
21 +template <typename scalar_t>
22 __global__ void rms_norm_kernel(
23 - scalar_t* __restrict__ out, // [..., hidden_size]
24 - const scalar_t* __restrict__ input, // [..., hidden_size]
25 - const scalar_t* __restrict__ weight, // [hidden_size]
26 - const float epsilon,
27 - const int num_tokens,
28 - const int hidden_size) {
23 + scalar_t* __restrict__ out, // [..., hidden_size]
24 + const scalar_t* __restrict__ input, // [..., hidden_size]
25 + const scalar_t* __restrict__ weight, // [hidden_size]
26 + const float epsilon, const int num_tokens, const int hidden_size) {
27 __shared__ float s_variance;
28 float variance = 0.0f;
29
30 for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
33 - const float x = (float) input[blockIdx.x * hidden_size + idx];
31 + const float x = (float)input[blockIdx.x * hidden_size + idx];
32 variance += x * x;
33 }
34 variance = blockReduceSum<float>(variance);
@@ -40,12 +38,12 @@ __global__ void rms_norm_kernel(
38 __syncthreads();
39
40 for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
43 - float x = (float) input[blockIdx.x * hidden_size + idx];
44 - out[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
41 + float x = (float)input[blockIdx.x * hidden_size + idx];
42 + out[blockIdx.x * hidden_size + idx] =
43 + ((scalar_t)(x * s_variance)) * weight[idx];
44 }
45 }
46
48 -
47 /* Converter structs for the conversion from torch types to HIP/CUDA types,
48 and the associated type conversions within HIP/CUDA. These helpers need
49 to be implemented for now because the relevant type conversion
@@ -54,51 +52,68 @@ __global__ void rms_norm_kernel(
52
53 Each struct should have the member static constexpr bool `exists`:
54 If false, the optimized kernel is not used for the corresponding torch type.
57 - If true, the struct should be fully defined as shown in the examples below.
55 + If true, the struct should be fully defined as shown in the examples below.
56 */
59 -template<typename torch_type>
60 -struct _typeConvert { static constexpr bool exists = false; };
57 +template <typename torch_type>
58 +struct _typeConvert {
59 + static constexpr bool exists = false;
60 +};
61
62 #if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
63 // CUDA < 12.0 runs into issues with packed type conversion
64 -template<>
64 +template <>
65 struct _typeConvert<c10::Half> {
66 static constexpr bool exists = true;
67 using hip_type = __half;
68 using packed_hip_type = __half2;
69
70 __device__ static inline float convert(hip_type x) { return __half2float(x); }
71 - __device__ static inline float2 convert(packed_hip_type x) { return __half22float2(x); }
72 - __device__ static inline hip_type convert(float x) { return __float2half_rn(x); }
73 - __device__ static inline packed_hip_type convert(float2 x) { return __float22half2_rn(x); }
71 + __device__ static inline float2 convert(packed_hip_type x) {
72 + return __half22float2(x);
73 + }
74 + __device__ static inline hip_type convert(float x) {
75 + return __float2half_rn(x);
76 + }
77 + __device__ static inline packed_hip_type convert(float2 x) {
78 + return __float22half2_rn(x);
79 + }
80 };
81
76 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
82 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
83 // CUDA_ARCH < 800 does not have BF16 support
84 // TODO: Add in ROCm support once public headers handle bf16 maturely
79 -template<>
85 +template <>
86 struct _typeConvert<c10::BFloat16> {
87 static constexpr bool exists = true;
88 using hip_type = __nv_bfloat16;
89 using packed_hip_type = __nv_bfloat162;
90
85 - __device__ static inline float convert(hip_type x) { return __bfloat162float(x); }
86 - __device__ static inline float2 convert(packed_hip_type x) { return __bfloat1622float2(x); }
87 - __device__ static inline hip_type convert(float x) { return __float2bfloat16(x); }
88 - __device__ static inline packed_hip_type convert(float2 x) { return __float22bfloat162_rn(x); }
91 + __device__ static inline float convert(hip_type x) {
92 + return __bfloat162float(x);
93 + }
94 + __device__ static inline float2 convert(packed_hip_type x) {
95 + return __bfloat1622float2(x);
96 + }
97 + __device__ static inline hip_type convert(float x) {
98 + return __float2bfloat16(x);
99 + }
100 + __device__ static inline packed_hip_type convert(float2 x) {
101 + return __float22bfloat162_rn(x);
102 + }
103 };
90 -#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
91 -#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
104 + #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
105 +#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
106 + // 12000))
107
108 /* Vector POD struct to generate vectorized and packed FP16/BF16 ops
109 for appropriate specializations of fused_add_rms_norm_kernel.
110 Only functions that are necessary in that kernel are implemented.
111 Alignment to 16 bytes is required to use 128-bit global memory ops.
112 */
98 -template<typename scalar_t, int width>
113 +template <typename scalar_t, int width>
114 struct alignas(16) _f16Vec {
100 - /* Not theoretically necessary that width is a power of 2 but should
101 - almost always be the case for optimization purposes */
115 + /* Not theoretically necessary that width is a power of 2 but should
116 + almost always be the case for optimization purposes */
117 static_assert(width > 0 && (width & (width - 1)) == 0,
118 "Width is not a positive power of 2!");
119 using Converter = _typeConvert<scalar_t>;
@@ -108,51 +123,49 @@ struct alignas(16) _f16Vec {
123
124 __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
125 if constexpr (width % 2 == 0) {
111 - #pragma unroll
126 +#pragma unroll
127 for (int i = 0; i < width; i += 2) {
113 - T2 temp{data[i], data[i+1]};
114 - temp += T2{other.data[i], other.data[i+1]};
128 + T2 temp{data[i], data[i + 1]};
129 + temp += T2{other.data[i], other.data[i + 1]};
130 data[i] = temp.x;
116 - data[i+1] = temp.y;
131 + data[i + 1] = temp.y;
132 }
133 } else {
119 - #pragma unroll
120 - for (int i = 0; i < width; ++i)
121 - data[i] += other.data[i];
134 +#pragma unroll
135 + for (int i = 0; i < width; ++i) data[i] += other.data[i];
136 }
137 return *this;
138 }
139
140 __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
141 if constexpr (width % 2 == 0) {
128 - #pragma unroll
142 +#pragma unroll
143 for (int i = 0; i < width; i += 2) {
130 - T2 temp{data[i], data[i+1]};
131 - temp *= T2{other.data[i], other.data[i+1]};
144 + T2 temp{data[i], data[i + 1]};
145 + temp *= T2{other.data[i], other.data[i + 1]};
146 data[i] = temp.x;
133 - data[i+1] = temp.y;
147 + data[i + 1] = temp.y;
148 }
149 } else {
136 - #pragma unroll
137 - for (int i = 0; i < width; ++i)
138 - data[i] *= other.data[i];
150 +#pragma unroll
151 + for (int i = 0; i < width; ++i) data[i] *= other.data[i];
152 }
153 return *this;
154 }
155
156 __device__ _f16Vec& operator*=(const float scale) {
157 if constexpr (width % 2 == 0) {
145 - #pragma unroll
158 +#pragma unroll
159 for (int i = 0; i < width; i += 2) {
147 - float2 temp_f = Converter::convert(T2{data[i], data[i+1]});
160 + float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
161 temp_f.x *= scale;
162 temp_f.y *= scale;
163 T2 temp = Converter::convert(temp_f);
164 data[i] = temp.x;
152 - data[i+1] = temp.y;
165 + data[i + 1] = temp.y;
166 }
167 } else {
155 - #pragma unroll
168 +#pragma unroll
169 for (int i = 0; i < width; ++i) {
170 float temp = Converter::convert(data[i]) * scale;
171 data[i] = Converter::convert(temp);
@@ -164,13 +177,13 @@ struct alignas(16) _f16Vec {
177 __device__ float sum_squares() const {
178 float result = 0.0f;
179 if constexpr (width % 2 == 0) {
167 - #pragma unroll
180 +#pragma unroll
181 for (int i = 0; i < width; i += 2) {
169 - float2 z = Converter::convert(T2{data[i], data[i+1]});
182 + float2 z = Converter::convert(T2{data[i], data[i + 1]});
183 result += z.x * z.x + z.y * z.y;
184 }
185 } else {
173 - #pragma unroll
186 +#pragma unroll
187 for (int i = 0; i < width; ++i) {
188 float x = Converter::convert(data[i]);
189 result += x * x;
@@ -184,15 +197,13 @@ struct alignas(16) _f16Vec {
197 Additional optimizations we can make in this case are
198 packed and vectorized operations, which help with the
199 memory latency bottleneck. */
187 -template<typename scalar_t, int width>
188 -__global__ std::enable_if_t<
189 - (width > 0) && _typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
190 - scalar_t* __restrict__ input, // [..., hidden_size]
191 - scalar_t* __restrict__ residual, // [..., hidden_size]
192 - const scalar_t* __restrict__ weight, // [hidden_size]
193 - const float epsilon,
194 - const int num_tokens,
195 - const int hidden_size) {
200 +template <typename scalar_t, int width>
201 +__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
202 +fused_add_rms_norm_kernel(
203 + scalar_t* __restrict__ input, // [..., hidden_size]
204 + scalar_t* __restrict__ residual, // [..., hidden_size]
205 + const scalar_t* __restrict__ weight, // [hidden_size]
206 + const float epsilon, const int num_tokens, const int hidden_size) {
207 // Sanity checks on our vector struct and type-punned pointer arithmetic
208 static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
209 static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
@@ -203,9 +214,12 @@ __global__ std::enable_if_t<
214 /* These and the argument pointers are all declared `restrict` as they are
215 not aliased in practice. Argument pointers should not be dereferenced
216 in this kernel as that would be undefined behavior */
206 - auto* __restrict__ input_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
207 - auto* __restrict__ residual_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
208 - auto* __restrict__ weight_v = reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
217 + auto* __restrict__ input_v =
218 + reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
219 + auto* __restrict__ residual_v =
220 + reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
221 + auto* __restrict__ weight_v =
222 + reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
223
224 for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
225 int id = blockIdx.x * vec_hidden_size + idx;
@@ -215,10 +229,11 @@ __global__ std::enable_if_t<
229 residual_v[id] = temp;
230 }
231 /* Keep the following if-else block in sync with the
218 - calculation of max_block_size in fused_add_rms_norm */
232 + calculation of max_block_size in fused_add_rms_norm */
233 if (num_tokens < 256) {
234 variance = blockReduceSum<float, 1024>(variance);
221 - } else variance = blockReduceSum<float, 256>(variance);
235 + } else
236 + variance = blockReduceSum<float, 256>(variance);
237 if (threadIdx.x == 0) {
238 s_variance = rsqrtf(variance / hidden_size + epsilon);
239 }
@@ -233,52 +248,50 @@ __global__ std::enable_if_t<
248 }
249 }
250
236 -
251 /* Generic fused_add_rms_norm_kernel
252 The width field is not used here but necessary for other specializations.
253 */
240 -template<typename scalar_t, int width>
241 -__global__ std::enable_if_t<
242 - (width == 0) || !_typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
243 - scalar_t* __restrict__ input, // [..., hidden_size]
244 - scalar_t* __restrict__ residual, // [..., hidden_size]
245 - const scalar_t* __restrict__ weight, // [hidden_size]
246 - const float epsilon,
247 - const int num_tokens,
248 - const int hidden_size) {
254 +template <typename scalar_t, int width>
255 +__global__ std::enable_if_t<(width == 0) || !_typeConvert<scalar_t>::exists>
256 +fused_add_rms_norm_kernel(
257 + scalar_t* __restrict__ input, // [..., hidden_size]
258 + scalar_t* __restrict__ residual, // [..., hidden_size]
259 + const scalar_t* __restrict__ weight, // [hidden_size]
260 + const float epsilon, const int num_tokens, const int hidden_size) {
261 __shared__ float s_variance;
262 float variance = 0.0f;
263
264 for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
265 scalar_t z = input[blockIdx.x * hidden_size + idx];
266 z += residual[blockIdx.x * hidden_size + idx];
255 - float x = (float) z;
267 + float x = (float)z;
268 variance += x * x;
269 residual[blockIdx.x * hidden_size + idx] = z;
270 }
271 /* Keep the following if-else block in sync with the
260 - calculation of max_block_size in fused_add_rms_norm */
272 + calculation of max_block_size in fused_add_rms_norm */
273 if (num_tokens < 256) {
274 variance = blockReduceSum<float, 1024>(variance);
263 - } else variance = blockReduceSum<float, 256>(variance);
275 + } else
276 + variance = blockReduceSum<float, 256>(variance);
277 if (threadIdx.x == 0) {
278 s_variance = rsqrtf(variance / hidden_size + epsilon);
279 }
280 __syncthreads();
281
282 for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
270 - float x = (float) residual[blockIdx.x * hidden_size + idx];
271 - input[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
283 + float x = (float)residual[blockIdx.x * hidden_size + idx];
284 + input[blockIdx.x * hidden_size + idx] =
285 + ((scalar_t)(x * s_variance)) * weight[idx];
286 }
287 }
288
275 -} // namespace vllm
289 +} // namespace vllm
290
277 -void rms_norm(
278 - torch::Tensor& out, // [..., hidden_size]
279 - torch::Tensor& input, // [..., hidden_size]
280 - torch::Tensor& weight, // [hidden_size]
281 - float epsilon) {
291 +void rms_norm(torch::Tensor& out, // [..., hidden_size]
292 + torch::Tensor& input, // [..., hidden_size]
293 + torch::Tensor& weight, // [hidden_size]
294 + float epsilon) {
295 int hidden_size = input.size(-1);
296 int num_tokens = input.numel() / hidden_size;
297
@@ -286,40 +299,27 @@ void rms_norm(
299 dim3 block(std::min(hidden_size, 1024));
300 const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
301 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
289 - VLLM_DISPATCH_FLOATING_TYPES(
290 - input.scalar_type(),
291 - "rms_norm_kernel",
292 - [&] {
293 - vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
294 - out.data_ptr<scalar_t>(),
295 - input.data_ptr<scalar_t>(),
296 - weight.data_ptr<scalar_t>(),
297 - epsilon,
298 - num_tokens,
299 - hidden_size);
300 - });
302 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_kernel", [&] {
303 + vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
304 + out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
305 + weight.data_ptr<scalar_t>(), epsilon, num_tokens, hidden_size);
306 + });
307 }
308
303 -#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
304 - VLLM_DISPATCH_FLOATING_TYPES( \
305 - input.scalar_type(), \
306 - "fused_add_rms_norm_kernel", \
307 - [&] { \
308 - vllm::fused_add_rms_norm_kernel \
309 - <scalar_t, width><<<grid, block, 0, stream>>>( \
310 - input.data_ptr<scalar_t>(), \
311 - residual.data_ptr<scalar_t>(), \
312 - weight.data_ptr<scalar_t>(), \
313 - epsilon, \
314 - num_tokens, \
315 - hidden_size); \
316 - });
317 -
318 -void fused_add_rms_norm(
319 - torch::Tensor& input, // [..., hidden_size]
320 - torch::Tensor& residual, // [..., hidden_size]
321 - torch::Tensor& weight, // [hidden_size]
322 - float epsilon) {
309 +#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
310 + VLLM_DISPATCH_FLOATING_TYPES( \
311 + input.scalar_type(), "fused_add_rms_norm_kernel", [&] { \
312 + vllm::fused_add_rms_norm_kernel<scalar_t, width> \
313 + <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(), \
314 + residual.data_ptr<scalar_t>(), \
315 + weight.data_ptr<scalar_t>(), epsilon, \
316 + num_tokens, hidden_size); \
317 + });
318 +
319 +void fused_add_rms_norm(torch::Tensor& input, // [..., hidden_size]
320 + torch::Tensor& residual, // [..., hidden_size]
321 + torch::Tensor& weight, // [hidden_size]
322 + float epsilon) {
323 int hidden_size = input.size(-1);
324 int num_tokens = input.numel() / hidden_size;
325
@@ -342,8 +342,8 @@ void fused_add_rms_norm(
342 auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
343 auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
344 auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
345 - bool ptrs_are_aligned = inp_ptr % 16 == 0 && res_ptr % 16 == 0 \
346 - && wt_ptr % 16 == 0;
345 + bool ptrs_are_aligned =
346 + inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
347 if (ptrs_are_aligned && hidden_size % 8 == 0) {
348 LAUNCH_FUSED_ADD_RMS_NORM(8);
349 } else {
csrc/moe/moe_ops.cpp
+2 -1
@@ -3,5 +3,6 @@
3 #include <torch/extension.h>
4
5 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6 - m.def("topk_softmax", &topk_softmax, "Apply topk softmax to the gating outputs.");
6 + m.def("topk_softmax", &topk_softmax,
7 + "Apply topk softmax to the gating outputs.");
8 }
csrc/moe/moe_ops.h
+3 -5
@@ -2,8 +2,6 @@
2
3 #include <torch/extension.h>
4
5 -void topk_softmax(
6 - torch::Tensor& topk_weights,
7 - torch::Tensor& topk_indices,
8 - torch::Tensor& token_expert_indices,
9 - torch::Tensor& gating_output);
5 +void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6 + torch::Tensor& token_expert_indices,
7 + torch::Tensor& gating_output);
csrc/moe_align_block_size_kernels.cu
+110 -101
@@ -7,119 +7,128 @@
7 #include "cuda_compat.h"
8 #include "dispatch_utils.h"
9
10 -#define CEILDIV(x,y) (((x) + (y) - 1) / (y))
10 +#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
11
12 namespace vllm {
13
14 namespace {
15 -__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row, int32_t col) {
16 - // don't worry about overflow because num_experts is relatively small
17 - return row * total_col + col;
18 -}
15 +__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row,
16 + int32_t col) {
17 + // don't worry about overflow because num_experts is relatively small
18 + return row * total_col + col;
19 }
20 +} // namespace
21
22 template <typename scalar_t>
22 -__global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
23 - int32_t *sorted_token_ids,
24 - int32_t *expert_ids,
25 - int32_t *total_tokens_post_pad,
26 - int32_t num_experts,
27 - int32_t block_size,
28 - size_t numel) {
29 - const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30 - const size_t start_idx = threadIdx.x * tokens_per_thread;
31 -
32 - extern __shared__ int32_t shared_mem[];
33 -
34 - int32_t* tokens_cnts = shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
35 - int32_t* cumsum = shared_mem + (num_experts + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
36 -
37 - for (int i = 0; i < num_experts; ++i) {
38 - tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
39 - }
40 -
41 - /**
42 - * In the first step we compute token_cnts[thread_index + 1][expert_index],
43 - * which counts how many tokens in the token shard of thread_index are assigned
44 - * to expert expert_index.
45 - */
46 - for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
47 - ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
48 - }
49 -
50 - __syncthreads();
51 -
52 - // For each expert we accumulate the token counts from the different threads.
53 - tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
54 - for (int i = 1; i <= blockDim.x; ++i) {
55 - tokens_cnts[index(num_experts, i, threadIdx.x)] += tokens_cnts[index(num_experts, i-1, threadIdx.x)];
56 - }
57 -
58 - __syncthreads();
59 -
60 - // We accumulate the token counts of all experts in thread 0.
61 - if (threadIdx.x == 0) {
62 - cumsum[0] = 0;
63 - for (int i = 1; i <= num_experts; ++i) {
64 - cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)], block_size) * block_size;
65 - }
66 - *total_tokens_post_pad = cumsum[num_experts];
67 - }
68 -
69 - __syncthreads();
70 -
71 - /**
72 - * For each expert, each thread processes the tokens of the corresponding blocks
73 - * and stores the corresponding expert_id for each block.
74 - */
75 - for (int i = cumsum[threadIdx.x];i < cumsum[threadIdx.x + 1];i += block_size) {
76 - expert_ids[i / block_size] = threadIdx.x;
23 +__global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
24 + int32_t* sorted_token_ids,
25 + int32_t* expert_ids,
26 + int32_t* total_tokens_post_pad,
27 + int32_t num_experts,
28 + int32_t block_size, size_t numel) {
29 + const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30 + const size_t start_idx = threadIdx.x * tokens_per_thread;
31 +
32 + extern __shared__ int32_t shared_mem[];
33 +
34 + int32_t* tokens_cnts =
35 + shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
36 + int32_t* cumsum =
37 + shared_mem + (num_experts + 1) *
38 + num_experts; // 1d tensor with shape (num_experts + 1)
39 +
40 + for (int i = 0; i < num_experts; ++i) {
41 + tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
42 + }
43 +
44 + /**
45 + * In the first step we compute token_cnts[thread_index + 1][expert_index],
46 + * which counts how many tokens in the token shard of thread_index are
47 + * assigned to expert expert_index.
48 + */
49 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
50 + ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
51 + }
52 +
53 + __syncthreads();
54 +
55 + // For each expert we accumulate the token counts from the different threads.
56 + tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
57 + for (int i = 1; i <= blockDim.x; ++i) {
58 + tokens_cnts[index(num_experts, i, threadIdx.x)] +=
59 + tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
60 + }
61 +
62 + __syncthreads();
63 +
64 + // We accumulate the token counts of all experts in thread 0.
65 + if (threadIdx.x == 0) {
66 + cumsum[0] = 0;
67 + for (int i = 1; i <= num_experts; ++i) {
68 + cumsum[i] = cumsum[i - 1] +
69 + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)],
70 + block_size) *
71 + block_size;
72 }
78 -
79 - /**
80 - * Each thread processes a token shard, calculating the index of each token after
81 - * sorting by expert number. Given the example topk_ids = [0,1,2,1,2,3,0,3,4] and
82 - * block_size = 4, then the output would be [0, 6, *, *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *],
83 - * where * represents a padding value(preset in python).
84 - */
85 - for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
86 - int32_t expert_id = topk_ids[i];
87 - /** The cumsum[expert_id] stores the starting index of the tokens that the
88 - * expert with expert_id needs to process, and tokens_cnts[threadIdx.x][expert_id]
89 - * stores the indices of the tokens processed by the expert with expert_id within
90 - * the current thread's token shard.
91 - */
92 - int32_t rank_post_pad = tokens_cnts[index(num_experts, threadIdx.x, expert_id)] + cumsum[expert_id];
93 - sorted_token_ids[rank_post_pad] = i;
94 - ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
95 - }
96 -}
73 + *total_tokens_post_pad = cumsum[num_experts];
74 + }
75 +
76 + __syncthreads();
77 +
78 + /**
79 + * For each expert, each thread processes the tokens of the corresponding
80 + * blocks and stores the corresponding expert_id for each block.
81 + */
82 + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
83 + i += block_size) {
84 + expert_ids[i / block_size] = threadIdx.x;
85 + }
86 +
87 + /**
88 + * Each thread processes a token shard, calculating the index of each token
89 + * after sorting by expert number. Given the example topk_ids =
90 + * [0,1,2,1,2,3,0,3,4] and block_size = 4, then the output would be [0, 6, *,
91 + * *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *], where * represents a
92 + * padding value(preset in python).
93 + */
94 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
95 + int32_t expert_id = topk_ids[i];
96 + /** The cumsum[expert_id] stores the starting index of the tokens that the
97 + * expert with expert_id needs to process, and
98 + * tokens_cnts[threadIdx.x][expert_id] stores the indices of the tokens
99 + * processed by the expert with expert_id within the current thread's token
100 + * shard.
101 + */
102 + int32_t rank_post_pad =
103 + tokens_cnts[index(num_experts, threadIdx.x, expert_id)] +
104 + cumsum[expert_id];
105 + sorted_token_ids[rank_post_pad] = i;
106 + ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
107 + }
108 }
98 -
99 -void moe_align_block_size(
100 - torch::Tensor topk_ids,
101 - int num_experts,
102 - int block_size,
103 - torch::Tensor sorted_token_ids,
104 - torch::Tensor experts_ids,
105 - torch::Tensor num_tokens_post_pad) {
106 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
107 - VLLM_DISPATCH_INTEGRAL_TYPES(
108 - topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
109 - // calc needed amount of shared mem for `tokens_cnts` and `cumsum` tensors
110 - const int32_t shared_mem = ((num_experts + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
109 +} // namespace vllm
110 +
111 +void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
112 + int block_size, torch::Tensor sorted_token_ids,
113 + torch::Tensor experts_ids,
114 + torch::Tensor num_tokens_post_pad) {
115 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
116 + VLLM_DISPATCH_INTEGRAL_TYPES(
117 + topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
118 + // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
119 + // tensors
120 + const int32_t shared_mem =
121 + ((num_experts + 1) * num_experts + (num_experts + 1)) *
122 + sizeof(int32_t);
123
124 // set dynamic shared mem
125 auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114 - AT_CUDA_CHECK(
115 - VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize((void *)kernel, shared_mem));
126 + AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
127 + (void*)kernel, shared_mem));
128 kernel<<<1, num_experts, shared_mem, stream>>>(
117 - topk_ids.data_ptr<scalar_t>(),
118 - sorted_token_ids.data_ptr<int32_t>(),
119 - experts_ids.data_ptr<int32_t>(),
120 - num_tokens_post_pad.data_ptr<int32_t>(),
121 - num_experts,
122 - block_size,
129 + topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
130 + experts_ids.data_ptr<int32_t>(),
131 + num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
132 topk_ids.numel());
124 - });
133 + });
134 }
csrc/ops.h
+121 -209
@@ -2,224 +2,136 @@
2
3 #include <torch/extension.h>
4
5 -void paged_attention_v1(
6 - torch::Tensor& out,
7 - torch::Tensor& query,
8 - torch::Tensor& key_cache,
9 - torch::Tensor& value_cache,
10 - int num_kv_heads,
11 - float scale,
12 - torch::Tensor& block_tables,
13 - torch::Tensor& seq_lens,
14 - int block_size,
15 - int max_seq_len,
16 - const c10::optional<torch::Tensor>& alibi_slopes,
17 - const std::string& kv_cache_dtype,
18 - float kv_scale);
19 -
20 -void paged_attention_v2(
21 - torch::Tensor& out,
22 - torch::Tensor& exp_sums,
23 - torch::Tensor& max_logits,
24 - torch::Tensor& tmp_out,
25 - torch::Tensor& query,
26 - torch::Tensor& key_cache,
27 - torch::Tensor& value_cache,
28 - int num_kv_heads,
29 - float scale,
30 - torch::Tensor& block_tables,
31 - torch::Tensor& seq_lens,
32 - int block_size,
33 - int max_seq_len,
34 - const c10::optional<torch::Tensor>& alibi_slopes,
35 - const std::string& kv_cache_dtype,
36 - float kv_scale);
37 -
38 -void rms_norm(
39 - torch::Tensor& out,
40 - torch::Tensor& input,
41 - torch::Tensor& weight,
42 - float epsilon);
43 -
44 -void fused_add_rms_norm(
45 - torch::Tensor& input,
46 - torch::Tensor& residual,
47 - torch::Tensor& weight,
48 - float epsilon);
49 -
50 -void rotary_embedding(
51 - torch::Tensor& positions,
52 - torch::Tensor& query,
53 - torch::Tensor& key,
54 - int head_size,
55 - torch::Tensor& cos_sin_cache,
56 - bool is_neox);
57 -
58 -void batched_rotary_embedding(
59 - torch::Tensor& positions,
60 - torch::Tensor& query,
61 - torch::Tensor& key,
62 - int head_size,
63 - torch::Tensor& cos_sin_cache,
64 - bool is_neox,
65 - int rot_dim,
66 - torch::Tensor& cos_sin_cache_offsets);
67 -
68 -void silu_and_mul(
69 - torch::Tensor& out,
70 - torch::Tensor& input);
71 -
72 -void gelu_and_mul(
73 - torch::Tensor& out,
74 - torch::Tensor& input);
75 -
76 -void gelu_tanh_and_mul(
77 - torch::Tensor& out,
78 - torch::Tensor& input);
79 -
80 -void gelu_new(
81 - torch::Tensor& out,
82 - torch::Tensor& input);
83 -
84 -void gelu_fast(
85 - torch::Tensor& out,
86 - torch::Tensor& input);
5 +void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
6 + torch::Tensor& key_cache, torch::Tensor& value_cache,
7 + int num_kv_heads, float scale,
8 + torch::Tensor& block_tables, torch::Tensor& seq_lens,
9 + int block_size, int max_seq_len,
10 + const c10::optional<torch::Tensor>& alibi_slopes,
11 + const std::string& kv_cache_dtype, float kv_scale);
12 +
13 +void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
14 + torch::Tensor& max_logits, torch::Tensor& tmp_out,
15 + torch::Tensor& query, torch::Tensor& key_cache,
16 + torch::Tensor& value_cache, int num_kv_heads,
17 + float scale, torch::Tensor& block_tables,
18 + torch::Tensor& seq_lens, int block_size,
19 + int max_seq_len,
20 + const c10::optional<torch::Tensor>& alibi_slopes,
21 + const std::string& kv_cache_dtype, float kv_scale);
22 +
23 +void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
24 + float epsilon);
25 +
26 +void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
27 + torch::Tensor& weight, float epsilon);
28 +
29 +void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
30 + torch::Tensor& key, int head_size,
31 + torch::Tensor& cos_sin_cache, bool is_neox);
32 +
33 +void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
34 + torch::Tensor& key, int head_size,
35 + torch::Tensor& cos_sin_cache, bool is_neox,
36 + int rot_dim,
37 + torch::Tensor& cos_sin_cache_offsets);
38 +
39 +void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
40 +
41 +void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
42 +
43 +void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
44 +
45 +void gelu_new(torch::Tensor& out, torch::Tensor& input);
46 +
47 +void gelu_fast(torch::Tensor& out, torch::Tensor& input);
48
49 #ifndef USE_ROCM
89 -torch::Tensor aqlm_gemm(
90 - const torch::Tensor& input,
91 - const torch::Tensor& codes,
92 - const torch::Tensor& codebooks,
93 - const torch::Tensor& scales,
94 - const torch::Tensor& codebook_partition_sizes,
95 - const std::optional<torch::Tensor>& bias
96 -);
97 -
98 -torch::Tensor aqlm_dequant(
99 - const torch::Tensor& codes,
100 - const torch::Tensor& codebooks,
101 - const torch::Tensor& codebook_partition_sizes
102 -);
103 -
104 -torch::Tensor awq_gemm(
105 - torch::Tensor _in_feats,
106 - torch::Tensor _kernel,
107 - torch::Tensor _scaling_factors,
108 - torch::Tensor _zeros,
109 - int split_k_iters);
110 -
111 -torch::Tensor awq_dequantize(
112 - torch::Tensor _kernel,
113 - torch::Tensor _scaling_factors,
114 - torch::Tensor _zeros,
115 - int split_k_iters,
116 - int thx,
117 - int thy);
118 -
119 -torch::Tensor marlin_gemm(
120 - torch::Tensor& a,
121 - torch::Tensor& b_q_weight,
122 - torch::Tensor& b_scales,
123 - torch::Tensor& workspace,
124 - int64_t size_m,
125 - int64_t size_n,
126 - int64_t size_k);
127 -
128 -torch::Tensor gptq_marlin_24_gemm(
129 - torch::Tensor &a,
130 - torch::Tensor &b_q_weight,
131 - torch::Tensor &b_meta,
132 - torch::Tensor &b_scales,
133 - torch::Tensor &workspace,
134 - int64_t num_bits,
135 - int64_t size_m,
136 - int64_t size_n,
137 - int64_t size_k);
138 -
139 -torch::Tensor gptq_marlin_gemm(
140 - torch::Tensor &a,
141 - torch::Tensor &b_q_weight,
142 - torch::Tensor &b_scales,
143 - torch::Tensor &g_idx,
144 - torch::Tensor &perm,
145 - torch::Tensor &workspace,
146 - int64_t num_bits,
147 - int64_t size_m,
148 - int64_t size_n,
149 - int64_t size_k,
150 - bool is_k_full);
151 -
152 -torch::Tensor gptq_marlin_repack(
153 - torch::Tensor &b_q_weight,
154 - torch::Tensor &perm,
155 - int64_t size_k,
156 - int64_t size_n,
157 - int64_t num_bits);
158 -
159 -int cutlass_scaled_mm_dq(
160 - torch::Tensor& out,
161 - torch::Tensor const &a,
162 - torch::Tensor const &b,
163 - torch::Tensor const &a_scales,
164 - torch::Tensor const &b_scales);
50 +torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
51 + const torch::Tensor& codebooks,
52 + const torch::Tensor& scales,
53 + const torch::Tensor& codebook_partition_sizes,
54 + const std::optional<torch::Tensor>& bias);
55 +
56 +torch::Tensor aqlm_dequant(const torch::Tensor& codes,
57 + const torch::Tensor& codebooks,
58 + const torch::Tensor& codebook_partition_sizes);
59 +
60 +torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
61 + torch::Tensor _scaling_factors, torch::Tensor _zeros,
62 + int split_k_iters);
63 +
64 +torch::Tensor awq_dequantize(torch::Tensor _kernel,
65 + torch::Tensor _scaling_factors,
66 + torch::Tensor _zeros, int split_k_iters, int thx,
67 + int thy);
68 +
69 +torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
70 + torch::Tensor& b_scales, torch::Tensor& workspace,
71 + int64_t size_m, int64_t size_n, int64_t size_k);
72 +
73 +torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
74 + torch::Tensor& b_meta,
75 + torch::Tensor& b_scales,
76 + torch::Tensor& workspace, int64_t num_bits,
77 + int64_t size_m, int64_t size_n,
78 + int64_t size_k);
79 +
80 +torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
81 + torch::Tensor& b_scales, torch::Tensor& g_idx,
82 + torch::Tensor& perm, torch::Tensor& workspace,
83 + int64_t num_bits, int64_t size_m, int64_t size_n,
84 + int64_t size_k, bool is_k_full);
85 +
86 +torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
87 + int64_t size_k, int64_t size_n,
88 + int64_t num_bits);
89 +
90 +int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
91 + torch::Tensor const& b, torch::Tensor const& a_scales,
92 + torch::Tensor const& b_scales);
93
94 #endif
95
168 -void squeezellm_gemm(
169 - torch::Tensor vec,
170 - torch::Tensor mat,
171 - torch::Tensor mul,
172 - torch::Tensor lookup_table);
173 -
174 -torch::Tensor gptq_gemm(
175 - torch::Tensor a,
176 - torch::Tensor b_q_weight,
177 - torch::Tensor b_gptq_qzeros,
178 - torch::Tensor b_gptq_scales,
179 - torch::Tensor b_g_idx,
180 - bool use_exllama,
181 - int bit);
182 -
183 -void gptq_shuffle(
184 - torch::Tensor q_weight,
185 - torch::Tensor q_perm,
186 - int bit);
187 -
188 -void static_scaled_fp8_quant(
189 - torch::Tensor& out,
190 - torch::Tensor& input,
191 - torch::Tensor& scale);
192 -
193 -void dynamic_scaled_fp8_quant(
194 - torch::Tensor& out,
195 - torch::Tensor& input,
196 - torch::Tensor& scale);
197 -
198 -void moe_align_block_size(
199 - torch::Tensor topk_ids,
200 - int num_experts,
201 - int block_size,
202 - torch::Tensor sorted_token_ids,
203 - torch::Tensor experts_ids,
204 - torch::Tensor num_tokens_post_pad);
96 +void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
97 + torch::Tensor lookup_table);
98 +
99 +torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
100 + torch::Tensor b_gptq_qzeros,
101 + torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
102 + bool use_exllama, int bit);
103 +
104 +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit);
105 +
106 +void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
107 + torch::Tensor& scale);
108 +
109 +void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
110 + torch::Tensor& scale);
111 +
112 +void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
113 + int block_size, torch::Tensor sorted_token_ids,
114 + torch::Tensor experts_ids,
115 + torch::Tensor num_tokens_post_pad);
116
117 #ifndef USE_ROCM
118 using fptr_t = uint64_t;
208 -fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
209 - const std::vector<std::string> &handles,
210 - const std::vector<int64_t> &offsets, int rank,
211 - bool full_nvlink);
212 -bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
119 +fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
120 + const std::vector<std::string>& handles,
121 + const std::vector<int64_t>& offsets, int rank,
122 + bool full_nvlink);
123 +bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
124 bool full_nvlink);
214 -void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out);
215 -void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &reg_buffer,
216 - torch::Tensor &out);
125 +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
126 +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
127 + torch::Tensor& out);
128 void dispose(fptr_t _fa);
129 int meta_size();
219 -void register_buffer(fptr_t _fa, torch::Tensor &t,
220 - const std::vector<std::string> &handles,
221 - const std::vector<int64_t> &offsets);
222 -std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(fptr_t _fa);
223 -void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
224 - const std::vector<std::vector<int64_t>> &offsets);
130 +void register_buffer(fptr_t _fa, torch::Tensor& t,
131 + const std::vector<std::string>& handles,
132 + const std::vector<int64_t>& offsets);
133 +std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
134 + fptr_t _fa);
135 +void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
136 + const std::vector<std::vector<int64_t>>& offsets);
137 #endif
csrc/pos_encoding_kernels.cu
+103 -126
@@ -7,14 +7,10 @@
7
8 namespace vllm {
9
10 -template<typename scalar_t, bool IS_NEOX>
10 +template <typename scalar_t, bool IS_NEOX>
11 inline __device__ void apply_token_rotary_embedding(
12 - scalar_t* __restrict__ arr,
13 - const scalar_t* __restrict__ cos_ptr,
14 - const scalar_t* __restrict__ sin_ptr,
15 - int rot_offset,
16 - int embed_dim)
17 -{
12 + scalar_t* __restrict__ arr, const scalar_t* __restrict__ cos_ptr,
13 + const scalar_t* __restrict__ sin_ptr, int rot_offset, int embed_dim) {
14 int x_index, y_index;
15 scalar_t cos, sin;
16 if (IS_NEOX) {
@@ -37,19 +33,17 @@ inline __device__ void apply_token_rotary_embedding(
33 arr[y_index] = y * cos + x * sin;
34 }
35
40 -template<typename scalar_t, bool IS_NEOX>
36 +template <typename scalar_t, bool IS_NEOX>
37 inline __device__ void apply_rotary_embedding(
42 - scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
43 - scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
44 - const scalar_t* cache_ptr,
45 - const int head_size,
46 - const int num_heads,
47 - const int num_kv_heads,
48 - const int rot_dim,
49 - const int token_idx,
50 - const int64_t query_stride,
51 - const int64_t key_stride)
52 -{
38 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
39 + // head_size] or [num_tokens, num_heads,
40 + // head_size]
41 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
42 + // head_size] or [num_tokens, num_kv_heads,
43 + // head_size]
44 + const scalar_t* cache_ptr, const int head_size, const int num_heads,
45 + const int num_kv_heads, const int rot_dim, const int token_idx,
46 + const int64_t query_stride, const int64_t key_stride) {
47 const int embed_dim = rot_dim / 2;
48 const scalar_t* cos_ptr = cache_ptr;
49 const scalar_t* sin_ptr = cache_ptr + embed_dim;
@@ -59,8 +53,8 @@ inline __device__ void apply_rotary_embedding(
53 const int head_idx = i / embed_dim;
54 const int64_t token_head = token_idx * query_stride + head_idx * head_size;
55 const int rot_offset = i % embed_dim;
62 - apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
63 - sin_ptr, rot_offset, embed_dim);
56 + apply_token_rotary_embedding<scalar_t, IS_NEOX>(
57 + query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
58 }
59
60 const int nk = num_kv_heads * embed_dim;
@@ -68,62 +62,74 @@ inline __device__ void apply_rotary_embedding(
62 const int head_idx = i / embed_dim;
63 const int64_t token_head = token_idx * key_stride + head_idx * head_size;
64 const int rot_offset = i % embed_dim;
71 - apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
72 - sin_ptr, rot_offset, embed_dim);
65 + apply_token_rotary_embedding<scalar_t, IS_NEOX>(
66 + key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
67 }
68 }
69
76 -template<typename scalar_t, bool IS_NEOX>
70 +template <typename scalar_t, bool IS_NEOX>
71 __global__ void rotary_embedding_kernel(
78 - const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
79 - scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
80 - scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
81 - const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
82 - const int rot_dim,
83 - const int64_t query_stride,
84 - const int64_t key_stride,
85 - const int num_heads,
86 - const int num_kv_heads,
87 - const int head_size) {
72 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or
73 + // [num_tokens]
74 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
75 + // head_size] or [num_tokens, num_heads,
76 + // head_size]
77 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
78 + // head_size] or [num_tokens, num_kv_heads,
79 + // head_size]
80 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
81 + // 2]
82 + const int rot_dim, const int64_t query_stride, const int64_t key_stride,
83 + const int num_heads, const int num_kv_heads, const int head_size) {
84 // Each thread block is responsible for one token.
85 const int token_idx = blockIdx.x;
86 int64_t pos = positions[token_idx];
87 const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
88
93 - apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
89 + apply_rotary_embedding<scalar_t, IS_NEOX>(
90 + query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim,
91 + token_idx, query_stride, key_stride);
92 }
93
96 -template<typename scalar_t, bool IS_NEOX>
94 +template <typename scalar_t, bool IS_NEOX>
95 __global__ void batched_rotary_embedding_kernel(
98 - const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99 - scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
100 - scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
101 - const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
102 - const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len] or [num_tokens]
103 - const int rot_dim,
104 - const int64_t query_stride,
105 - const int64_t key_stride,
106 - const int num_heads,
107 - const int num_kv_heads,
108 - const int head_size) {
96 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or
97 + // [num_tokens]
98 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
99 + // head_size] or [num_tokens, num_heads,
100 + // head_size]
101 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
102 + // head_size] or [num_tokens, num_kv_heads,
103 + // head_size]
104 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
105 + // 2]
106 + const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len]
107 + // or [num_tokens]
108 + const int rot_dim, const int64_t query_stride, const int64_t key_stride,
109 + const int num_heads, const int num_kv_heads, const int head_size) {
110 // Each thread block is responsible for one token.
111 const int token_idx = blockIdx.x;
112 int64_t pos = positions[token_idx];
113 int64_t cos_sin_cache_offset = cos_sin_cache_offsets[token_idx];
113 - const scalar_t* cache_ptr = cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
114 + const scalar_t* cache_ptr =
115 + cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
116
115 - apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
117 + apply_rotary_embedding<scalar_t, IS_NEOX>(
118 + query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim,
119 + token_idx, query_stride, key_stride);
120 }
121
118 -} // namespace vllm
122 +} // namespace vllm
123
124 void rotary_embedding(
121 - torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
122 - torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
123 - torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
124 - int head_size,
125 - torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
126 - bool is_neox) {
125 + torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
126 + torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
127 + // [num_tokens, num_heads * head_size]
128 + torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
129 + // [num_tokens, num_kv_heads * head_size]
130 + int head_size,
131 + torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
132 + bool is_neox) {
133 int64_t num_tokens = query.numel() / query.size(-1);
134 int rot_dim = cos_sin_cache.size(1);
135 int num_heads = query.size(-1) / head_size;
@@ -135,36 +141,21 @@ void rotary_embedding(
141 dim3 block(std::min(num_heads * rot_dim / 2, 512));
142 const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
143 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
138 - VLLM_DISPATCH_FLOATING_TYPES(
139 - query.scalar_type(),
140 - "rotary_embedding",
141 - [&] {
142 - if (is_neox) {
143 - vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
144 - positions.data_ptr<int64_t>(),
145 - query.data_ptr<scalar_t>(),
146 - key.data_ptr<scalar_t>(),
147 - cos_sin_cache.data_ptr<scalar_t>(),
148 - rot_dim,
149 - query_stride,
150 - key_stride,
151 - num_heads,
152 - num_kv_heads,
153 - head_size);
154 - } else {
155 - vllm::rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
156 - positions.data_ptr<int64_t>(),
157 - query.data_ptr<scalar_t>(),
158 - key.data_ptr<scalar_t>(),
159 - cos_sin_cache.data_ptr<scalar_t>(),
160 - rot_dim,
161 - query_stride,
162 - key_stride,
163 - num_heads,
164 - num_kv_heads,
165 - head_size);
166 - }
167 - });
144 + VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
145 + if (is_neox) {
146 + vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
147 + positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
148 + key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(), rot_dim,
149 + query_stride, key_stride, num_heads, num_kv_heads, head_size);
150 + } else {
151 + vllm::rotary_embedding_kernel<scalar_t, false>
152 + <<<grid, block, 0, stream>>>(
153 + positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
154 + key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
155 + rot_dim, query_stride, key_stride, num_heads, num_kv_heads,
156 + head_size);
157 + }
158 + });
159 }
160
161 /*
@@ -172,14 +163,15 @@ Batched version of rotary embedding, pack multiple LoRAs together
163 and process in batched manner.
164 */
165 void batched_rotary_embedding(
175 - torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
176 - torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
177 - torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
178 - int head_size,
179 - torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
180 - bool is_neox,
181 - int rot_dim,
182 - torch::Tensor& cos_sin_cache_offsets // [num_tokens]
166 + torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
167 + torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
168 + // [num_tokens, num_heads * head_size]
169 + torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
170 + // [num_tokens, num_kv_heads * head_size]
171 + int head_size,
172 + torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
173 + bool is_neox, int rot_dim,
174 + torch::Tensor& cos_sin_cache_offsets // [num_tokens]
175 ) {
176 int64_t num_tokens = cos_sin_cache_offsets.size(0);
177 int num_heads = query.size(-1) / head_size;
@@ -191,36 +183,21 @@ void batched_rotary_embedding(
183 dim3 block(std::min(num_heads * rot_dim / 2, 512));
184 const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
185 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
194 - VLLM_DISPATCH_FLOATING_TYPES(
195 - query.scalar_type(),
196 - "rotary_embedding",
197 - [&] {
198 - if (is_neox) {
199 - vllm::batched_rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
200 - positions.data_ptr<int64_t>(),
201 - query.data_ptr<scalar_t>(),
202 - key.data_ptr<scalar_t>(),
203 - cos_sin_cache.data_ptr<scalar_t>(),
204 - cos_sin_cache_offsets.data_ptr<int64_t>(),
205 - rot_dim,
206 - query_stride,
207 - key_stride,
208 - num_heads,
209 - num_kv_heads,
210 - head_size);
211 - } else {
212 - vllm::batched_rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
213 - positions.data_ptr<int64_t>(),
214 - query.data_ptr<scalar_t>(),
215 - key.data_ptr<scalar_t>(),
216 - cos_sin_cache.data_ptr<scalar_t>(),
217 - cos_sin_cache_offsets.data_ptr<int64_t>(),
218 - rot_dim,
219 - query_stride,
220 - key_stride,
221 - num_heads,
222 - num_kv_heads,
223 - head_size);
224 - }
225 - });
186 + VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
187 + if (is_neox) {
188 + vllm::batched_rotary_embedding_kernel<scalar_t, true>
189 + <<<grid, block, 0, stream>>>(
190 + positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
191 + key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
192 + cos_sin_cache_offsets.data_ptr<int64_t>(), rot_dim, query_stride,
193 + key_stride, num_heads, num_kv_heads, head_size);
194 + } else {
195 + vllm::batched_rotary_embedding_kernel<scalar_t, false>
196 + <<<grid, block, 0, stream>>>(
197 + positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
198 + key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
199 + cos_sin_cache_offsets.data_ptr<int64_t>(), rot_dim, query_stride,
200 + key_stride, num_heads, num_kv_heads, head_size);
201 + }
202 + });
203 }
csrc/pybind.cpp
+56 -86
@@ -8,116 +8,87 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8 pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
10 // Attention ops
11 - ops.def(
12 - "paged_attention_v1",
13 - &paged_attention_v1,
14 - "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15 - ops.def(
16 - "paged_attention_v2",
17 - &paged_attention_v2,
18 - "PagedAttention V2.");
11 + ops.def("paged_attention_v1", &paged_attention_v1,
12 + "Compute the attention between an input query and the cached "
13 + "keys/values using PagedAttention.");
14 + ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15
16 // Activation ops
21 - ops.def(
22 - "silu_and_mul",
23 - &silu_and_mul,
24 - "Activation function used in SwiGLU.");
25 - ops.def(
26 - "gelu_and_mul",
27 - &gelu_and_mul,
28 - "Activation function used in GeGLU with `none` approximation.");
29 - ops.def(
30 - "gelu_tanh_and_mul",
31 - &gelu_tanh_and_mul,
32 - "Activation function used in GeGLU with `tanh` approximation.");
33 - ops.def(
34 - "gelu_new",
35 - &gelu_new,
36 - "GELU implementation used in GPT-2.");
37 - ops.def(
38 - "gelu_fast",
39 - &gelu_fast,
40 - "Approximate GELU implementation.");
17 + ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18 + ops.def("gelu_and_mul", &gelu_and_mul,
19 + "Activation function used in GeGLU with `none` approximation.");
20 + ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21 + "Activation function used in GeGLU with `tanh` approximation.");
22 + ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23 + ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24
25 // Layernorm
43 - ops.def(
44 - "rms_norm",
45 - &rms_norm,
46 - "Apply Root Mean Square (RMS) Normalization to the input tensor.");
26 + ops.def("rms_norm", &rms_norm,
27 + "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28
48 - ops.def(
49 - "fused_add_rms_norm",
50 - &fused_add_rms_norm,
51 - "In-place fused Add and RMS Normalization");
29 + ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30 + "In-place fused Add and RMS Normalization");
31
32 // Rotary embedding
54 - ops.def(
55 - "rotary_embedding",
56 - &rotary_embedding,
57 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
33 + ops.def("rotary_embedding", &rotary_embedding,
34 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35
59 - ops.def(
60 - "batched_rotary_embedding",
61 - &batched_rotary_embedding,
62 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key (supports multiple loras)");
36 + ops.def("batched_rotary_embedding", &batched_rotary_embedding,
37 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key "
38 + "(supports multiple loras)");
39
40 // Quantization ops
41 #ifndef USE_ROCM
42 ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
43 ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
44 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69 - ops.def("marlin_gemm", &marlin_gemm, "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
70 - ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm, "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
71 - ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72 - ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
45 + ops.def("marlin_gemm", &marlin_gemm,
46 + "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
47 + ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm,
48 + "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
49 + ops.def("gptq_marlin_gemm", &gptq_marlin_gemm,
50 + "gptq_marlin Optimized Quantized GEMM for GPTQ");
51 + ops.def("gptq_marlin_repack", &gptq_marlin_repack,
52 + "gptq_marlin repack from GPTQ");
53 ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
74 - ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq, "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column quantization.");
54 + ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq,
55 + "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or "
56 + "per-row/column quantization.");
57 #endif
76 -
58 +
59 ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
60 ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
61 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
80 - ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant, "Compute FP8 quantized tensor for given scaling factor");
81 - ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant, "Compute FP8 quantized tensor and scaling factor");
82 - ops.def(
83 - "moe_align_block_size",
84 - &moe_align_block_size,
85 - "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
62 + ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant,
63 + "Compute FP8 quantized tensor for given scaling factor");
64 + ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant,
65 + "Compute FP8 quantized tensor and scaling factor");
66 + ops.def("moe_align_block_size", &moe_align_block_size,
67 + "Aligning the number of tokens to be processed by each expert such "
68 + "that it is divisible by the block size.");
69
70 // Cache ops
71 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
89 - cache_ops.def(
90 - "swap_blocks",
91 - &swap_blocks,
92 - "Swap in (out) the cache blocks from src to dst");
93 - cache_ops.def(
94 - "copy_blocks",
95 - &copy_blocks,
96 - "Copy the cache blocks from src to dst");
97 - cache_ops.def(
98 - "reshape_and_cache",
99 - &reshape_and_cache,
100 - "Reshape the key and value tensors and cache them");
101 - cache_ops.def(
102 - "reshape_and_cache_flash",
103 - &reshape_and_cache_flash,
104 - "Reshape the key and value tensors and cache them");
105 - cache_ops.def(
106 - "convert_fp8",
107 - &convert_fp8,
108 - "Convert the key and value cache to fp8 data type");
72 + cache_ops.def("swap_blocks", &swap_blocks,
73 + "Swap in (out) the cache blocks from src to dst");
74 + cache_ops.def("copy_blocks", &copy_blocks,
75 + "Copy the cache blocks from src to dst");
76 + cache_ops.def("reshape_and_cache", &reshape_and_cache,
77 + "Reshape the key and value tensors and cache them");
78 + cache_ops.def("reshape_and_cache_flash", &reshape_and_cache_flash,
79 + "Reshape the key and value tensors and cache them");
80 + cache_ops.def("convert_fp8", &convert_fp8,
81 + "Convert the key and value cache to fp8 data type");
82
83 // Cuda utils
111 - pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
112 - cuda_utils.def(
113 - "get_device_attribute",
114 - &get_device_attribute,
115 - "Gets the specified device attribute.");
84 + pybind11::module cuda_utils =
85 + m.def_submodule("cuda_utils", "vLLM cuda utils");
86 + cuda_utils.def("get_device_attribute", &get_device_attribute,
87 + "Gets the specified device attribute.");
88
117 - cuda_utils.def(
118 - "get_max_shared_memory_per_block_device_attribute",
119 - &get_max_shared_memory_per_block_device_attribute,
120 - "Gets the maximum shared memory per block device attribute.");
89 + cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
90 + &get_max_shared_memory_per_block_device_attribute,
91 + "Gets the maximum shared memory per block device attribute.");
92
93 #ifndef USE_ROCM
94 // Custom all-reduce kernels
@@ -134,5 +105,4 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
105 custom_ar.def("register_graph_buffers", &register_graph_buffers,
106 "register_graph_buffers");
107 #endif
137 -
108 }
csrc/quantization/aqlm/gemm_kernels.cu
+211 -325
@@ -25,32 +25,28 @@
25 #include <iostream>
26 #include <cstdlib>
27
28 -
28 namespace vllm {
29 namespace aqlm {
30
31 __global__ void Code1x16MatVec(
33 - const int4* __restrict__ A,
34 - const int4* __restrict__ B,
35 - int4* __restrict__ C,
36 - const int4* __restrict__ codebook,
37 - const int prob_m,
38 - const int prob_k,
39 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
40 - const int codebook_stride // as int4.
32 + const int4* __restrict__ A, const int4* __restrict__ B,
33 + int4* __restrict__ C, const int4* __restrict__ codebook, const int prob_m,
34 + const int prob_k,
35 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each
36 + // codebook, at most 3 long.
37 + const int codebook_stride // as int4.
38 ) {
39 int a_gl_stride = prob_k / 8 / 8;
40 int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
41 bool pred = a_gl_rd < prob_m;
42
46 - if (pred)
47 - {
48 - // advance to the correct codebook, this easy because we only multiply one column of the codebook.
43 + if (pred) {
44 + // advance to the correct codebook, this easy because we only multiply one
45 + // column of the codebook.
46 auto codebook_size = &codebook_a_sizes.x;
50 - while (a_gl_rd >= *codebook_size)
51 - {
52 - codebook += codebook_stride;
53 - ++codebook_size;
47 + while (a_gl_rd >= *codebook_size) {
48 + codebook += codebook_stride;
49 + ++codebook_size;
50 }
51 }
52
@@ -67,8 +63,7 @@ __global__ void Code1x16MatVec(
63 // We pad shared memory to avoid bank conflicts during reads
64 __syncthreads();
65 for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
70 - if (b_gl_rd + i < prob_k / 8)
71 - sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
66 + if (b_gl_rd + i < prob_k / 8) sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
67 }
68 __syncthreads();
69 b_gl_rd += 32 * 8;
@@ -76,22 +71,19 @@ __global__ void Code1x16MatVec(
71 int b_sh_rd = 9 * (threadIdx.x % 32);
72 if (pred && a_gl_rd < a_gl_end) {
73 const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
79 - #pragma unroll
74 +#pragma unroll
75 for (int i = 0; i < 8; i++) {
76 uint32_t dec[4];
82 - // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
83 - // actually help us; this brings > 2x speedup.
84 - asm volatile (
85 - "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
86 - : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
87 - : "l"((void*) &codebook[enc[i]])
88 - );
77 + // We bypass the L1 cache to avoid massive amounts of memory streaming
78 + // that doesn't actually help us; this brings > 2x speedup.
79 + asm volatile("ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
80 + : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
81 + : "l"((void*)&codebook[enc[i]]));
82 half2* a = reinterpret_cast<half2*>(&dec);
83 half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
84 half2 res2 = {};
92 - #pragma unroll
93 - for (int j = 0; j < 4; j++)
94 - res2 = __hfma2(a[j], b[j], res2);
85 +#pragma unroll
86 + for (int j = 0; j < 4; j++) res2 = __hfma2(a[j], b[j], res2);
87 res += __half2float(res2.x) + __half2float(res2.y);
88 b_sh_rd++;
89 }
@@ -100,37 +92,33 @@ __global__ void Code1x16MatVec(
92 }
93
94 if (pred) {
103 - #pragma unroll
104 - for (int i = 16; i > 0; i /= 2)
105 - res += __shfl_down_sync(0xffffffff, res, i);
95 +#pragma unroll
96 + for (int i = 16; i > 0; i /= 2) res += __shfl_down_sync(0xffffffff, res, i);
97 if (threadIdx.x % 32 == 0)
98 reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
99 }
100 }
101
102 __global__ void Code2x8MatVec(
112 - const int4* __restrict__ A,
113 - const int4* __restrict__ B,
114 - int4* __restrict__ C,
115 - const int4* __restrict__ codebook,
116 - int prob_m,
117 - int prob_k,
118 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
119 - const int codebook_stride // as int4.
103 + const int4* __restrict__ A, const int4* __restrict__ B,
104 + int4* __restrict__ C, const int4* __restrict__ codebook, int prob_m,
105 + int prob_k,
106 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each
107 + // codebook, at most 3 long.
108 + const int codebook_stride // as int4.
109
110 ) {
111 int a_gl_stride = prob_k / 8 / 8;
112 int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
113 bool pred = a_gl_rd < prob_m;
114
126 - if (pred)
127 - {
128 - // advance to the correct codebook, this easy because we only multiply one column of the codebook.
115 + if (pred) {
116 + // advance to the correct codebook, this easy because we only multiply one
117 + // column of the codebook.
118 auto codebook_size = &codebook_a_sizes.x;
130 - while (a_gl_rd >= *codebook_size)
131 - {
132 - codebook += codebook_stride;
133 - ++codebook_size;
119 + while (a_gl_rd >= *codebook_size) {
120 + codebook += codebook_stride;
121 + ++codebook_size;
122 }
123 }
124
@@ -148,9 +136,8 @@ __global__ void Code2x8MatVec(
136
137 for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
138 int4 dec = codebook[i];
151 - #pragma unroll
152 - for (int j = 0; j < 8; j++)
153 - sh_code[8 * i + (j + lane) % 8] = dec;
139 +#pragma unroll
140 + for (int j = 0; j < 8; j++) sh_code[8 * i + (j + lane) % 8] = dec;
141 }
142 __syncthreads();
143
@@ -161,8 +148,7 @@ __global__ void Code2x8MatVec(
148 // We pad shared memory to avoid bank conflicts during reads
149 __syncthreads();
150 for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
164 - if (b_gl_rd + i < prob_k / 8)
165 - sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
151 + if (b_gl_rd + i < prob_k / 8) sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
152 }
153 __syncthreads();
154 b_gl_rd += 32 * 8;
@@ -170,13 +156,15 @@ __global__ void Code2x8MatVec(
156 int b_sh_rd = 9 * (threadIdx.x % 32);
157 if (pred && a_gl_rd < a_gl_end) {
158 const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
173 - #pragma unroll
159 +#pragma unroll
160 for (int i = 0; i < 8; i++) {
175 - half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
176 - half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
177 - half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
161 + half2* a0 =
162 + reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
163 + half2* a1 =
164 + reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
165 + half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
166 half2 res2 = {};
179 - #pragma unroll
167 +#pragma unroll
168 for (int j = 0; j < 4; j++)
169 res2 = __hfma2(__hadd2(a0[j], a1[j]), b[j], res2);
170 res += __half2float(res2.x) + __half2float(res2.y);
@@ -187,36 +175,31 @@ __global__ void Code2x8MatVec(
175 }
176
177 if (pred) {
190 - #pragma unroll
191 - for (int i = 16; i > 0; i /= 2)
192 - res += __shfl_down_sync(0xffffffff, res, i);
178 +#pragma unroll
179 + for (int i = 16; i > 0; i /= 2) res += __shfl_down_sync(0xffffffff, res, i);
180 if (threadIdx.x % 32 == 0)
181 reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
182 }
183 }
184
198 -
185 __global__ void Code1x16Dequant(
200 - const int4* __restrict__ A,
201 - int4* __restrict__ C,
202 - const int4* __restrict__ codebook,
203 - int prob_m,
204 - int prob_k,
205 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, sums to m.
206 - const int codebook_stride // as int4
186 + const int4* __restrict__ A, int4* __restrict__ C,
187 + const int4* __restrict__ codebook, int prob_m, int prob_k,
188 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each
189 + // codebook, at most 3 long, sums to m.
190 + const int codebook_stride // as int4
191 ) {
192 int a_gl_stride = prob_k / 8 / 8;
193 int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
194 bool pred = a_gl_rd < prob_m;
195
212 - if (pred)
213 - {
214 - // advance to the correct codebook, this easy because we only multiply one column of the codebook.
196 + if (pred) {
197 + // advance to the correct codebook, this easy because we only multiply one
198 + // column of the codebook.
199 auto codebook_size = &codebook_a_sizes.x;
216 - while (a_gl_rd >= *codebook_size)
217 - {
218 - codebook += codebook_stride;
219 - ++codebook_size;
200 + while (a_gl_rd >= *codebook_size) {
201 + codebook += codebook_stride;
202 + ++codebook_size;
203 }
204 }
205
@@ -231,17 +214,15 @@ __global__ void Code1x16Dequant(
214 while (iters--) {
215 if (pred && a_gl_rd < a_gl_end) {
216 const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
234 - #pragma unroll
217 +#pragma unroll
218 for (int i = 0; i < 8; i++) {
219 int4 chunk;
220 auto dec = reinterpret_cast<uint32_t*>(&chunk);
238 - // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
239 - // actually help us; this brings > 2x speedup.
240 - asm volatile (
241 - "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
242 - : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
243 - : "l"((void*) &codebook[enc[i]])
244 - );
221 + // We bypass the L1 cache to avoid massive amounts of memory streaming
222 + // that doesn't actually help us; this brings > 2x speedup.
223 + asm volatile("ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
224 + : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
225 + : "l"((void*)&codebook[enc[i]]));
226
227 C[a_gl_rd * 8 + i] = chunk;
228 }
@@ -250,28 +231,25 @@ __global__ void Code1x16Dequant(
231 }
232 }
233
253 -
234 __global__ void Code2x8Dequant(
255 - const int4* __restrict__ A,
256 - int4* __restrict__ C,
257 - const int4* __restrict__ codebook,
258 - int prob_m,
259 - int prob_k,
260 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
261 - const int codebook_stride // as int4
235 + const int4* __restrict__ A, int4* __restrict__ C,
236 + const int4* __restrict__ codebook, int prob_m, int prob_k,
237 + const int4
238 + codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
239 + // most 3 long, corresponds to cols.
240 + const int codebook_stride // as int4
241 ) {
242 int a_gl_stride = prob_k / 8 / 8;
243 int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
244 bool pred = a_gl_rd < prob_m;
245
267 - if (pred)
268 - {
269 - // advance to the correct codebook, this easy because we only multiply one column of the codebook.
246 + if (pred) {
247 + // advance to the correct codebook, this easy because we only multiply one
248 + // column of the codebook.
249 auto codebook_size = &codebook_a_sizes.x;
271 - while (a_gl_rd >= *codebook_size)
272 - {
273 - codebook += codebook_stride;
274 - ++codebook_size;
250 + while (a_gl_rd >= *codebook_size) {
251 + codebook += codebook_stride;
252 + ++codebook_size;
253 }
254 }
255
@@ -290,9 +268,8 @@ __global__ void Code2x8Dequant(
268
269 for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
270 int4 dec = codebook[i];
293 - #pragma unroll
294 - for (int j = 0; j < 8; j++)
295 - sh_code[8 * i + (j + lane) % 8] = dec;
271 +#pragma unroll
272 + for (int j = 0; j < 8; j++) sh_code[8 * i + (j + lane) % 8] = dec;
273 }
274 __syncthreads();
275
@@ -302,12 +279,14 @@ __global__ void Code2x8Dequant(
279 while (iters--) {
280 if (pred && a_gl_rd < a_gl_end) {
281 const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
305 - #pragma unroll
282 +#pragma unroll
283 for (int i = 0; i < 8; i++) {
284 int4 chunk;
308 - half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
309 - half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
310 - #pragma unroll
285 + half2* a0 =
286 + reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
287 + half2* a1 =
288 + reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
289 +#pragma unroll
290 for (int j = 0; j < 4; j++)
291 reinterpret_cast<half2*>(&chunk)[j] = __hadd2(a0[j], a1[j]);
292 C[a_gl_rd * 8 + i] = chunk;
@@ -317,22 +296,15 @@ __global__ void Code2x8Dequant(
296 }
297 }
298
320 -inline int ceildiv(int a, int b) {
321 - return (a + b - 1) / b;
322 -}
299 +inline int ceildiv(int a, int b) { return (a + b - 1) / b; }
300
301 const int THREAD_M = 16;
302
326 -void code1x16_matvec_cuda(
327 - const void* __restrict__ A,
328 - const void* __restrict__ B,
329 - void* __restrict__ C,
330 - const void* __restrict__ codebook,
331 - int prob_m,
332 - int prob_k,
333 - const int4 codebook_a_sizes,
334 - const int codebook_stride
335 -) {
303 +void code1x16_matvec_cuda(const void* __restrict__ A,
304 + const void* __restrict__ B, void* __restrict__ C,
305 + const void* __restrict__ codebook, int prob_m,
306 + int prob_k, const int4 codebook_a_sizes,
307 + const int codebook_stride) {
308 int sms;
309 cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
310 int waves = 0;
@@ -345,28 +317,16 @@ void code1x16_matvec_cuda(
317 int blocks = ceildiv(prob_m, thread_m);
318 int threads = 32 * thread_m;
319 cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
348 - Code1x16MatVec<<<blocks, threads, 16*32*9, stream>>>(
349 - (const int4*) A,
350 - (const int4*) B,
351 - (int4*) C,
352 - (const int4*) codebook,
353 - prob_m,
354 - prob_k,
355 - codebook_a_sizes,
356 - codebook_stride
357 - );
320 + Code1x16MatVec<<<blocks, threads, 16 * 32 * 9, stream>>>(
321 + (const int4*)A, (const int4*)B, (int4*)C, (const int4*)codebook, prob_m,
322 + prob_k, codebook_a_sizes, codebook_stride);
323 }
324
360 -void code2x8_matvec_cuda(
361 - const void* __restrict__ A,
362 - const void* __restrict__ B,
363 - void* __restrict__ C,
364 - const void* __restrict__ codebook,
365 - int prob_m,
366 - int prob_k,
367 - const int4 codebook_a_sizes,
368 - const int codebook_stride
369 -) {
325 +void code2x8_matvec_cuda(const void* __restrict__ A, const void* __restrict__ B,
326 + void* __restrict__ C,
327 + const void* __restrict__ codebook, int prob_m,
328 + int prob_k, const int4 codebook_a_sizes,
329 + const int codebook_stride) {
330 int sms;
331 cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
332 int waves = 0;
@@ -379,30 +339,20 @@ void code2x8_matvec_cuda(
339 int blocks = ceildiv(prob_m, thread_m);
340 int threads = 32 * thread_m;
341 int shared = 16 * (2 * 256 * 8 + 32 * 9);
382 - cudaFuncSetAttribute(
383 - Code2x8MatVec, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
384 - );
342 + cudaFuncSetAttribute(Code2x8MatVec,
343 + cudaFuncAttributeMaxDynamicSharedMemorySize, shared);
344 cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
345 Code2x8MatVec<<<blocks, threads, shared, stream>>>(
387 - (const int4*) A,
388 - (const int4*) B,
389 - (int4*) C,
390 - (const int4*) codebook,
391 - prob_m,
392 - prob_k,
393 - codebook_a_sizes,
394 - codebook_stride
395 - );
346 + (const int4*)A, (const int4*)B, (int4*)C, (const int4*)codebook, prob_m,
347 + prob_k, codebook_a_sizes, codebook_stride);
348 }
349
350 void code1x16_dequant_cuda(
399 - const void* __restrict__ A,
400 - void* __restrict__ C,
401 - const void* __restrict__ codebook,
402 - int prob_m,
403 - int prob_k,
404 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
405 - const int codebook_stride // as int4.
351 + const void* __restrict__ A, void* __restrict__ C,
352 + const void* __restrict__ codebook, int prob_m, int prob_k,
353 + const int4 codebook_a_sizes, // cumulative sizes of A spanning each
354 + // codebook, at most 3 long.
355 + const int codebook_stride // as int4.
356 ) {
357 int sms;
358 cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
@@ -417,25 +367,21 @@ void code1x16_dequant_cuda(
367 int threads = 32 * thread_m;
368 cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
369 Code1x16Dequant<<<blocks, threads, 0, stream>>>(
420 - (const int4*) A,
421 - (int4*) C,
422 - (const int4*) codebook,
423 - prob_m,
424 - prob_k,
425 - codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
426 - codebook_stride // as int4.
370 + (const int4*)A, (int4*)C, (const int4*)codebook, prob_m, prob_k,
371 + codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
372 + // most 3 long.
373 + codebook_stride // as int4.
374 );
375 }
376
377 // Dequantizes the code and codebook into weights.
431 -void code2x8_dequant_cuda(
432 - const void* __restrict__ A,
433 - void* __restrict__ C,
434 - const void* __restrict__ codebook,
435 - int prob_m,
436 - int prob_k,
437 - const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
438 - const int codebook_stride // as int4
378 +void code2x8_dequant_cuda(
379 + const void* __restrict__ A, void* __restrict__ C,
380 + const void* __restrict__ codebook, int prob_m, int prob_k,
381 + const int4
382 + codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
383 + // most 3 long, corresponds to cols.
384 + const int codebook_stride // as int4
385 ) {
386 int sms;
387 cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
@@ -451,74 +397,50 @@ void code2x8_dequant_cuda(
397 int shared = 16 * (2 * 256 * 8 + 32 * 9);
398 cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
399
454 - cudaFuncSetAttribute(
455 - Code2x8Dequant, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
456 - );
400 + cudaFuncSetAttribute(Code2x8Dequant,
401 + cudaFuncAttributeMaxDynamicSharedMemorySize, shared);
402 Code2x8Dequant<<<blocks, threads, shared, stream>>>(
458 - (const int4*) A,
459 - (int4*) C,
460 - (const int4*) codebook,
461 - prob_m,
462 - prob_k,
463 - codebook_a_sizes,
464 - codebook_stride
465 - );
403 + (const int4*)A, (int4*)C, (const int4*)codebook, prob_m, prob_k,
404 + codebook_a_sizes, codebook_stride);
405 }
406
468 -int codebook_stride(const torch::Tensor& codebooks)
469 -{
407 +int codebook_stride(const torch::Tensor& codebooks) {
408 return codebooks.stride(0) * codebooks.element_size() / sizeof(int4);
409 }
410
411 void code1x16_matvec(
474 - const torch::Tensor& A,
475 - const torch::Tensor& B,
476 - torch::Tensor& C,
477 - const torch::Tensor& codebook,
478 - const int4 codebook_a_sizes // cumulative sizes of A spanning each codebook, at most 3 long.
412 + const torch::Tensor& A, const torch::Tensor& B, torch::Tensor& C,
413 + const torch::Tensor& codebook,
414 + const int4 codebook_a_sizes // cumulative sizes of A spanning each
415 + // codebook, at most 3 long.
416 ) {
417 const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
418 int prob_m = C.size(0);
419 int prob_k = B.size(0);
420
484 - code1x16_matvec_cuda(
485 - A.data_ptr(),
486 - B.data_ptr(),
487 - C.data_ptr(),
488 - codebook.data_ptr(),
489 - prob_m,
490 - prob_k,
491 - codebook_a_sizes,
492 - codebook_stride(codebook)
493 - );
421 + code1x16_matvec_cuda(A.data_ptr(), B.data_ptr(), C.data_ptr(),
422 + codebook.data_ptr(), prob_m, prob_k, codebook_a_sizes,
423 + codebook_stride(codebook));
424 }
425
496 -torch::Tensor code1x16_matmat(
497 - const torch::Tensor& input,
498 - const torch::Tensor& codes,
499 - const torch::Tensor& codebooks,
500 - const torch::Tensor& scales,
501 - const int4 codebook_a_sizes,
502 - const std::optional<torch::Tensor>& bias) {
426 +torch::Tensor code1x16_matmat(const torch::Tensor& input,
427 + const torch::Tensor& codes,
428 + const torch::Tensor& codebooks,
429 + const torch::Tensor& scales,
430 + const int4 codebook_a_sizes,
431 + const std::optional<torch::Tensor>& bias) {
432 auto input_sizes = input.sizes();
433 auto out_features = codes.size(0) * codebooks.size(2);
434 auto flat_input = input.reshape({-1, input.size(-1)});
506 - auto flat_output = torch::empty({flat_input.size(0), out_features},
507 - torch::TensorOptions()
508 - .dtype(input.dtype())
509 - .device(input.device())
510 - );
435 + auto flat_output = torch::empty(
436 + {flat_input.size(0), out_features},
437 + torch::TensorOptions().dtype(input.dtype()).device(input.device()));
438
439 for (int i = 0; i < flat_input.size(0); ++i) {
440 auto input_vec = flat_input.index({i});
441 auto output_vec = flat_output.index({i});
515 - code1x16_matvec(
516 - codes.squeeze(2),
517 - input_vec,
518 - output_vec,
519 - codebooks,
520 - codebook_a_sizes
521 - );
442 + code1x16_matvec(codes.squeeze(2), input_vec, output_vec, codebooks,
443 + codebook_a_sizes);
444 }
445 flat_output *= scales.flatten().unsqueeze(0);
446
@@ -533,55 +455,35 @@ torch::Tensor code1x16_matmat(
455 return output;
456 }
457
536 -void code2x8_matvec(
537 - const torch::Tensor& A,
538 - const torch::Tensor& B,
539 - torch::Tensor& C,
540 - const torch::Tensor& codebook,
541 - const int4 codebook_a_sizes
542 -) {
458 +void code2x8_matvec(const torch::Tensor& A, const torch::Tensor& B,
459 + torch::Tensor& C, const torch::Tensor& codebook,
460 + const int4 codebook_a_sizes) {
461 const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
462 int prob_m = C.size(0);
463 int prob_k = B.size(0);
546 - code2x8_matvec_cuda(
547 - A.data_ptr(),
548 - B.data_ptr(),
549 - C.data_ptr(),
550 - codebook.data_ptr(),
551 - prob_m,
552 - prob_k,
553 - codebook_a_sizes,
554 - 2 * codebook_stride(codebook)
555 - );
464 + code2x8_matvec_cuda(A.data_ptr(), B.data_ptr(), C.data_ptr(),
465 + codebook.data_ptr(), prob_m, prob_k, codebook_a_sizes,
466 + 2 * codebook_stride(codebook));
467 }
468
558 -torch::Tensor code2x8_matmat(
559 - const torch::Tensor& input,
560 - const torch::Tensor& codes,
561 - const torch::Tensor& codebooks,
562 - const torch::Tensor& scales,
563 - const int4 codebook_a_sizes,
564 - const std::optional<torch::Tensor>& bias
565 -) {
469 +torch::Tensor code2x8_matmat(const torch::Tensor& input,
470 + const torch::Tensor& codes,
471 + const torch::Tensor& codebooks,
472 + const torch::Tensor& scales,
473 + const int4 codebook_a_sizes,
474 + const std::optional<torch::Tensor>& bias) {
475 auto input_sizes = input.sizes();
476 auto out_features = codes.size(0) * codebooks.size(2);
477 auto flat_input = input.reshape({-1, input.size(-1)});
569 - auto flat_output = torch::empty({flat_input.size(0), out_features},
570 - torch::TensorOptions()
571 - .dtype(input.dtype())
572 - .device(input.device())
573 - );
478 + auto flat_output = torch::empty(
479 + {flat_input.size(0), out_features},
480 + torch::TensorOptions().dtype(input.dtype()).device(input.device()));
481
482 for (int i = 0; i < flat_input.size(0); ++i) {
483 auto input_vec = flat_input.index({i});
484 auto output_vec = flat_output.index({i});
578 - code2x8_matvec(
579 - codes.squeeze(2),
580 - input_vec,
581 - output_vec,
582 - codebooks,
583 - codebook_a_sizes
584 - );
485 + code2x8_matvec(codes.squeeze(2), input_vec, output_vec, codebooks,
486 + codebook_a_sizes);
487 }
488 flat_output *= scales.flatten().unsqueeze(0);
489 if (bias.has_value()) {
@@ -596,64 +498,56 @@ torch::Tensor code2x8_matmat(
498 }
499
500 // Accumulate the partition sizes.
599 -int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes)
600 -{
501 +int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
502 int4 cumulative_sizes;
503 auto cumulative_size = &cumulative_sizes.x;
504 int i = 0;
505 int last = 0;
506 assert(codebook_partition_sizes.size(0) <= 4);
606 - for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size)
607 - {
507 + for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size) {
508 *cumulative_size = codebook_partition_sizes[i].item<int>() + last;
509 last = *cumulative_size;
510 }
511 // fill in the rest with unreachable.
612 - for (; i < 4; ++i, ++cumulative_size)
613 - {
614 - *cumulative_size = last*10;
512 + for (; i < 4; ++i, ++cumulative_size) {
513 + *cumulative_size = last * 10;
514 }
515 return cumulative_sizes;
516 }
517
619 -} // namespace aqlm
620 -} // namespace vllm
621 -
518 +} // namespace aqlm
519 +} // namespace vllm
520
623 -torch::Tensor aqlm_gemm(
624 - const torch::Tensor& input,
625 - const torch::Tensor& codes,
626 - const torch::Tensor& codebooks,
627 - const torch::Tensor& scales,
628 - const torch::Tensor& codebook_partition_sizes,
629 - const std::optional<torch::Tensor>& bias
630 -)
631 -{
632 - int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
521 +torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
522 + const torch::Tensor& codebooks,
523 + const torch::Tensor& scales,
524 + const torch::Tensor& codebook_partition_sizes,
525 + const std::optional<torch::Tensor>& bias) {
526 + int4 cumulative_sizes =
527 + vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
528
529 int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
530 int const entries = codebooks.size(1);
531
637 - if (nbooks == 1 && entries == (1 << 16))
638 - {
639 - return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
532 + if (nbooks == 1 && entries == (1 << 16)) {
533 + return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales,
534 + cumulative_sizes, bias);
535 }
641 - if (nbooks == 2 && entries == (1 << 8))
642 - {
643 - return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
536 + if (nbooks == 2 && entries == (1 << 8)) {
537 + return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales,
538 + cumulative_sizes, bias);
539 }
540
646 - TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
541 + TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries,
542 + " entries is not currently supported.")
543 return {};
544 }
545
650 -torch::Tensor aqlm_dequant(
651 - const torch::Tensor& codes,
652 - const torch::Tensor& codebooks,
653 - const torch::Tensor& codebook_partition_sizes
654 -)
655 -{
656 - int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
546 +torch::Tensor aqlm_dequant(const torch::Tensor& codes,
547 + const torch::Tensor& codebooks,
548 + const torch::Tensor& codebook_partition_sizes) {
549 + int4 cumulative_sizes =
550 + vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
551
552 int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
553 int const entries = codebooks.size(1);
@@ -668,45 +562,37 @@ torch::Tensor aqlm_dequant(
562 assert(out_features = codebook_partition_sizes.sum().item<int>());
563
564 auto weights = torch::empty({out_features, in_features},
671 - torch::TensorOptions()
672 - .dtype(codebooks.dtype())
673 - .device(codebooks.device())
674 - );
565 + torch::TensorOptions()
566 + .dtype(codebooks.dtype())
567 + .device(codebooks.device()));
568 +
569 + if (nbooks == 1 && entries == (1 << 16)) {
570 + vllm::aqlm::code1x16_dequant_cuda(codes.data_ptr(), weights.data_ptr(),
571 + codebooks.data_ptr(), out_features,
572 + in_features, cumulative_sizes,
573 + vllm::aqlm::codebook_stride(codebooks));
574
676 - if (nbooks == 1 && entries == (1 << 16))
677 - {
678 - vllm::aqlm::code1x16_dequant_cuda(
679 - codes.data_ptr(),
680 - weights.data_ptr(),
681 - codebooks.data_ptr(),
682 - out_features,
683 - in_features,
684 - cumulative_sizes,
685 - vllm::aqlm::codebook_stride(codebooks));
686 -
687 - // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation.)
688 - // weights *= scales.index({"...", 0, 0});
689 -
690 - return weights;
575 + // if you wanted to flip to scaling the weights, (though it's 30%-ish slower
576 + // and not consistent with gemv implementation.) weights *=
577 + // scales.index({"...", 0, 0});
578 +
579 + return weights;
580 }
581
693 - if (nbooks == 2 && entries == (1 << 8))
694 - {
695 - vllm::aqlm::code2x8_dequant_cuda(
696 - codes.data_ptr(),
697 - weights.data_ptr(),
698 - codebooks.data_ptr(),
699 - out_features,
700 - in_features,
701 - cumulative_sizes,
702 - vllm::aqlm::codebook_stride(codebooks));
703 -
704 - // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation)
705 - // weights *= scales.index({"...", 0, 0});
706 -
707 - return weights;
582 + if (nbooks == 2 && entries == (1 << 8)) {
583 + vllm::aqlm::code2x8_dequant_cuda(codes.data_ptr(), weights.data_ptr(),
584 + codebooks.data_ptr(), out_features,
585 + in_features, cumulative_sizes,
586 + vllm::aqlm::codebook_stride(codebooks));
587 +
588 + // if you wanted to flip to scaling the weights, (though it's 30%-ish slower
589 + // and not consistent with gemv implementation) weights *=
590 + // scales.index({"...", 0, 0});
591 +
592 + return weights;
593 }
594
710 - TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
595 + TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries,
596 + " entries is not currently supported.")
597 return {};
598 }
csrc/quantization/awq/dequantize.cuh
+76 -62
@@ -1,11 +1,11 @@
1 /*
2 Adapted from https://github.com/mit-han-lab/llm-awq
3 -Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
3 +Modified from NVIDIA FasterTransformer:
4 +https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
5 @article{lin2023awq,
5 - title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
6 - author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
7 - journal={arXiv},
8 - year={2023}
6 + title={AWQ: Activation-aware Weight Quantization for LLM Compression and
7 +Acceleration}, author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang,
8 +Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
9 }
10 */
11
@@ -14,74 +14,88 @@ Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransfor
14 namespace vllm {
15 namespace awq {
16
17 -__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
18 -{
17 +__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source) {
18 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
19 assert(false);
20 #else
22 - uint4 result;
21 + uint4 result;
22
24 - uint32_t* h = reinterpret_cast<uint32_t*>(&result);
25 - uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
23 + uint32_t* h = reinterpret_cast<uint32_t*>(&result);
24 + uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
25
27 - // First, we extract the i4s and construct an intermediate fp16 number.
28 - static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
29 - static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
30 - static constexpr uint32_t TOP_MASK = 0x00f000f0;
31 - static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
26 + // First, we extract the i4s and construct an intermediate fp16 number.
27 + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
28 + static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
29 + static constexpr uint32_t TOP_MASK = 0x00f000f0;
30 + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
31
33 - // Note that the entire sequence only requires 1 shift instruction. This is thanks to the register packing
34 - // format and the fact that we force our integers to be unsigned, and account for this in the fp16 subtractions.
35 - // In addition, I exploit the fact that sub and fma have the same throughput in order to convert elt_23 and
36 - // elt_67 to fp16 without having to shift them to the bottom bits before hand.
32 + // Note that the entire sequence only requires 1 shift instruction. This is
33 + // thanks to the register packing format and the fact that we force our
34 + // integers to be unsigned, and account for this in the fp16 subtractions. In
35 + // addition, I exploit the fact that sub and fma have the same throughput in
36 + // order to convert elt_23 and elt_67 to fp16 without having to shift them to
37 + // the bottom bits before hand.
38
38 - // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW dependency if we issue
39 - // immediately before required.
40 - const uint32_t top_i4s = i4s >> 8;
41 - // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
42 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
43 - : "=r"(h[0])
44 - : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
45 - // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
46 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
47 - : "=r"(h[1])
48 - : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
49 - // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
50 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
51 - : "=r"(h[2])
52 - : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
53 - // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
54 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
55 - : "=r"(h[3])
56 - : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
39 + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW
40 + // dependency if we issue immediately before required.
41 + const uint32_t top_i4s = i4s >> 8;
42 + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
43 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
44 + : "=r"(h[0])
45 + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
46 + "n"(immLut));
47 + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
48 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
49 + : "=r"(h[1])
50 + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
51 + "n"(immLut));
52 + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
53 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
54 + : "=r"(h[2])
55 + : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
56 + "n"(immLut));
57 + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
58 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
59 + : "=r"(h[3])
60 + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
61 + "n"(immLut));
62
58 - // I use inline PTX below because I am not sure if the compiler will emit float2half instructions if I use the
59 - // half2 ctor. In this case, I chose performance reliability over code readability.
63 + // I use inline PTX below because I am not sure if the compiler will emit
64 + // float2half instructions if I use the half2 ctor. In this case, I chose
65 + // performance reliability over code readability.
66
61 - // This is the half2 {1032, 1032} represented as an integer.
62 - // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
63 - // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
64 - static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
65 - // This is the half2 {1 / 16, 1 / 16} represented as an integer.
66 - static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
67 - // This is the half2 {-72, -72} represented as an integer.
68 - // static constexpr uint32_t NEG_72 = 0xd480d480;
69 - // Haotian: Let's use {-64, -64}.
70 - static constexpr uint32_t NEG_64 = 0xd400d400;
67 + // This is the half2 {1032, 1032} represented as an integer.
68 + // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
69 + // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
70 + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
71 + // This is the half2 {1 / 16, 1 / 16} represented as an integer.
72 + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
73 + // This is the half2 {-72, -72} represented as an integer.
74 + // static constexpr uint32_t NEG_72 = 0xd480d480;
75 + // Haotian: Let's use {-64, -64}.
76 + static constexpr uint32_t NEG_64 = 0xd400d400;
77
72 - // Finally, we construct the output numbers.
73 - // Convert elt_01
74 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
75 - // Convert elt_23
76 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
77 - // Convert elt_45
78 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
79 - // Convert elt_67
80 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
78 + // Finally, we construct the output numbers.
79 + // Convert elt_01
80 + asm volatile("sub.f16x2 %0, %1, %2;\n"
81 + : "=r"(h[0])
82 + : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
83 + // Convert elt_23
84 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
85 + : "=r"(h[1])
86 + : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
87 + // Convert elt_45
88 + asm volatile("sub.f16x2 %0, %1, %2;\n"
89 + : "=r"(h[2])
90 + : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
91 + // Convert elt_67
92 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
93 + : "=r"(h[3])
94 + : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
95
82 - return result;
96 + return result;
97 #endif
98 }
99
86 -} // namespace awq
87 -} // namespace vllm
100 +} // namespace awq
101 +} // namespace vllm
csrc/quantization/awq/gemm_kernels.cu
+357 -254
@@ -1,14 +1,12 @@
1 /*
2 Adapted from https://github.com/mit-han-lab/llm-awq
3 @article{lin2023awq,
4 - title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
5 - author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
6 - journal={arXiv},
7 - year={2023}
4 + title={AWQ: Activation-aware Weight Quantization for LLM Compression and
5 +Acceleration}, author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang,
6 +Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
7 }
8 */
9
11 -
10 #include <torch/extension.h>
11 #include <c10/cuda/CUDAGuard.h>
12
@@ -20,26 +18,20 @@ namespace vllm {
18 namespace awq {
19
20 // Pack two half values.
23 -static inline __device__ __host__ unsigned
24 -__pack_half2(const half x, const half y) {
25 - unsigned v0 = *((unsigned short *)&x);
26 - unsigned v1 = *((unsigned short *)&y);
21 +static inline __device__ __host__ unsigned __pack_half2(const half x,
22 + const half y) {
23 + unsigned v0 = *((unsigned short*)&x);
24 + unsigned v1 = *((unsigned short*)&y);
25 return (v1 << 16) | v0;
26 }
27
30 -template<int N>
31 -__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
32 - int G,
33 - int split_k_iters,
34 - half* __restrict__ A,
35 - int* __restrict__ B,
36 - half* __restrict__ scaling_factors,
37 - int* __restrict__ zeros,
38 - int M,
39 - int IC,
40 - int OC,
41 - half* __restrict__ C)
42 -{
28 +template <int N>
29 +__global__ void __launch_bounds__(64)
30 + gemm_forward_4bit_cuda_m16nXk32(int G, int split_k_iters,
31 + half* __restrict__ A, int* __restrict__ B,
32 + half* __restrict__ scaling_factors,
33 + int* __restrict__ zeros, int M, int IC,
34 + int OC, half* __restrict__ C) {
35 // Only support matrix n = 64 or 128
36 assert(N == 64 || N == 128);
37 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
@@ -70,43 +62,46 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
62 static constexpr int row_stride = 2 * 32 * 8 / N;
63 bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < N;
64 // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
73 - bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
65 + bool ld_A_flag =
66 + (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp +
67 + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
68 // bool wb_C_flag = (threadIdx.x / 4) < M;
69
76 - half* A_ptr = A
77 - + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
78 - + (((int)threadIdx.x) % (32 / 8)) * 8;
79 -
80 - int* B_ptr = B
81 - + ((int)threadIdx.y) * (OC / 8) * (256 / N)
82 - + (((int)threadIdx.x) / (N / 8)) * (OC / 8)
83 - + (((int)blockIdx_y) % j_factors1) * (N / 8)
84 - + (((int)threadIdx.x) % (N / 8)) * 1;
85 -// Why * 1 in the above line?
86 -
87 - half* A_shared_ptr = A_shared
88 - + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
89 - + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
90 - + (((int)threadIdx.x) % (32 / 8) ) * 8;
91 -
92 - half* B_shared_ptr = B_shared
93 - + ((int)threadIdx.y) * (row_stride / 2) * (N + 8)
94 - + (((int)threadIdx.x) / (N / 8)) * (N + 8)
95 - + (((int)threadIdx.x) % (N / 8)) * 8;
96 -
97 - int* zeros_ptr = zeros
98 - + (((int)blockIdx_y) % j_factors1) * (N / 8)
99 - + ((int)threadIdx.x) % (N / 8);
100 -
101 - half* scaling_factors_ptr = scaling_factors
102 - + (((int)blockIdx_y) % j_factors1) * N
103 - + (((int)threadIdx.x) % (N / 8)) * 8;
104 -
105 - half* C_ptr = C
106 - + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
107 - + (((int)blockIdx_y) % j_factors1) * N
108 - + ((int)threadIdx.y) * (N / 2)
109 - + (((int)threadIdx.x) % 4) * 2;
70 + half* A_ptr =
71 + A +
72 + (((int)blockIdx_y) / j_factors1 * 16 +
73 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) *
74 + IC +
75 + (((int)threadIdx.x) % (32 / 8)) * 8;
76 +
77 + int* B_ptr = B + ((int)threadIdx.y) * (OC / 8) * (256 / N) +
78 + (((int)threadIdx.x) / (N / 8)) * (OC / 8) +
79 + (((int)blockIdx_y) % j_factors1) * (N / 8) +
80 + (((int)threadIdx.x) % (N / 8)) * 1;
81 + // Why * 1 in the above line?
82 +
83 + half* A_shared_ptr = A_shared +
84 + ((int)threadIdx.y) * row_stride_warp * (32 + 8) +
85 + (((int)threadIdx.x) / (32 / 8)) * (32 + 8) +
86 + (((int)threadIdx.x) % (32 / 8)) * 8;
87 +
88 + half* B_shared_ptr = B_shared +
89 + ((int)threadIdx.y) * (row_stride / 2) * (N + 8) +
90 + (((int)threadIdx.x) / (N / 8)) * (N + 8) +
91 + (((int)threadIdx.x) % (N / 8)) * 8;
92 +
93 + int* zeros_ptr = zeros + (((int)blockIdx_y) % j_factors1) * (N / 8) +
94 + ((int)threadIdx.x) % (N / 8);
95 +
96 + half* scaling_factors_ptr = scaling_factors +
97 + (((int)blockIdx_y) % j_factors1) * N +
98 + (((int)threadIdx.x) % (N / 8)) * 8;
99 +
100 + half* C_ptr =
101 + C +
102 + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
103 + + (((int)blockIdx_y) % j_factors1) * N + ((int)threadIdx.y) * (N / 2) +
104 + (((int)threadIdx.x) % 4) * 2;
105
106 // preload s.f. and zeros
107 int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
@@ -115,57 +110,83 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
110 int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
111 __syncthreads();
112 // TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
118 - if (ld_A_flag)
119 - {
113 + if (ld_A_flag) {
114 *(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
121 - }
122 - else
123 - {
115 + } else {
116 *(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
117 }
118
119 // for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
120 uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
121 uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
130 - uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
122 + uint4 B_loaded_scale =
123 + *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
124 /*
132 - if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
133 - printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
125 + if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 &&
126 + threadIdx.y == 0){ printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x,
127 + B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x,
128 + B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
129 }
130 */
131 // uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
132 int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
133
134 for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < N / 16; ++ax0_ax1_fused_0) {
140 -
135 // B: 32 x 136 (128+8) float16
136 // each warp: 32 x 4
143 - // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
144 - // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
145 - // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
146 - uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
137 + // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus
138 + // zero -> WB UINT4
139 + // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) *
140 + // 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15)
141 + // * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 *
142 + // 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) *
143 + // 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) *
144 + // 8))); row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
145 + uint32_t B_loaded =
146 + *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
147 uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
148 - //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
148 + // uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N /
149 + // 8)) * 8);
150
150 - // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
151 + // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x
152 + // % (cta_N / 8)) * 8);
153 // - zero and * scale
152 - // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
153 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
154 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
155 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
156 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
157 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
158 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
159 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
160 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
154 + // TODO (Haotian): can save 4 assembly instructions if sormulate as deq =
155 + // q * scale - zero * scale.
156 + asm volatile("sub.f16x2 %0, %1, %2;\n"
157 + : "=r"(B_loaded_fp16.x)
158 + : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
159 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
160 + : "=r"(B_loaded_fp16.x)
161 + : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
162 + asm volatile("sub.f16x2 %0, %1, %2;\n"
163 + : "=r"(B_loaded_fp16.y)
164 + : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
165 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
166 + : "=r"(B_loaded_fp16.y)
167 + : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
168 + asm volatile("sub.f16x2 %0, %1, %2;\n"
169 + : "=r"(B_loaded_fp16.z)
170 + : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
171 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
172 + : "=r"(B_loaded_fp16.z)
173 + : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
174 + asm volatile("sub.f16x2 %0, %1, %2;\n"
175 + : "=r"(B_loaded_fp16.w)
176 + : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
177 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
178 + : "=r"(B_loaded_fp16.w)
179 + : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
180 /*
162 - if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
163 - printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
181 + if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 ==
182 + 0 && threadIdx.x == 17 && threadIdx.y == 0){ printf("[x] %X %X %X %X\n",
183 + B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
184 }
185 */
186
187 // write back
168 - *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) = B_loaded_fp16;
188 + *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) =
189 + B_loaded_fp16;
190 }
191 __syncthreads();
192
@@ -173,112 +194,179 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
194 {
195 unsigned int addr;
196 __asm__ __volatile__(
176 - "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
177 - : "=r"(addr)
178 - : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
179 - );
180 -
197 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, "
198 + "addr; }\n"
199 + : "=r"(addr)
200 + : "l"((void*)((&(A_shared[(k_0_1 * 16)])) +
201 + (((((int)threadIdx.x) & 15) * 40) +
202 + ((((int)threadIdx.x) >> 4) * 8)))));
203
204 __asm__ __volatile__(
183 - "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
184 - "{%0, %1, %2, %3}, [%4];\n"
185 - : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
186 - : "r"(addr)
187 - );
205 + "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
206 + "{%0, %1, %2, %3}, [%4];\n"
207 + : "=r"(((unsigned*)(A_shared_warp + 0))[0]),
208 + "=r"(((unsigned*)(A_shared_warp + 0))[1]),
209 + "=r"(((unsigned*)(A_shared_warp + 0))[2]),
210 + "=r"(((unsigned*)(A_shared_warp + 0))[3])
211 + : "r"(addr));
212 }
213
214 for (int ax1_0 = 0; ax1_0 < N / 32; ++ax1_0) {
215 {
216 unsigned int addr;
217 __asm__ __volatile__(
194 - "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
195 - : "=r"(addr)
196 - : "l"((void *)((&(B_shared[(((k_0_1 * (N * 16 + 128)) + (((int)threadIdx.y) * (N / 2))) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * (N + 8)) + ((((int)threadIdx.x) >> 4) * 8))))
197 - );
218 + "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, "
219 + "addr; }\n"
220 + : "=r"(addr)
221 + : "l"((void*)((&(B_shared[(((k_0_1 * (N * 16 + 128)) +
222 + (((int)threadIdx.y) * (N / 2))) +
223 + (ax1_0 * 16))])) +
224 + (((((int)threadIdx.x) & 15) * (N + 8)) +
225 + ((((int)threadIdx.x) >> 4) * 8)))));
226 __asm__ __volatile__(
199 - "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
200 - "{%0, %1, %2, %3}, [%4];\n"
201 - : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
202 - : "r"(addr)
203 - );
227 + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
228 + "{%0, %1, %2, %3}, [%4];\n"
229 + : "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[0]),
230 + "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[1]),
231 + "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[2]),
232 + "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[3])
233 + : "r"(addr));
234 }
235 }
236 for (int j_0_4 = 0; j_0_4 < N / 32; ++j_0_4) {
207 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
237 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
238 {
239 __asm__ __volatile__(
210 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
211 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
212 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
213 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
240 + "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
241 + "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
242 + : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
243 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
244 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
245 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
246 + : "r"(((unsigned*)(A_shared_warp + 0))[0]),
247 + "r"(((unsigned*)(A_shared_warp + 0))[1]),
248 + "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[0]),
249 + "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
250 + "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
251 + "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
252 + "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
253 }
254
255 {
256 __asm__ __volatile__(
218 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
219 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
220 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
221 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
257 + "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
258 + "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
259 + : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
260 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
261 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
262 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
263 + : "r"(((unsigned*)(A_shared_warp + 0))[0]),
264 + "r"(((unsigned*)(A_shared_warp + 0))[1]),
265 + "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]),
266 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
267 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
268 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
269 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
270 }
271
272 {
273 __asm__ __volatile__(
226 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
227 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
228 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
229 - : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
274 + "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
275 + "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
276 + : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
277 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
278 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
279 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
280 + : "r"(((unsigned*)(A_shared_warp + 0))[2]),
281 + "r"(((unsigned*)(A_shared_warp + 0))[3]),
282 + "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[1]),
283 + "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
284 + "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
285 + "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
286 + "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
287 }
288
289 {
290 __asm__ __volatile__(
234 - "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
235 - "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
236 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
237 - : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
291 + "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
292 + "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
293 + : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
294 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
295 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
296 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
297 + : "r"(((unsigned*)(A_shared_warp + 0))[2]),
298 + "r"(((unsigned*)(A_shared_warp + 0))[3]),
299 + "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]),
300 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
301 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
302 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
303 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
304 }
239 -#else
305 + #else
306 {
307 __asm__ __volatile__(
242 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
243 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
244 - : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
245 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
308 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
309 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, "
310 + "%13};\n"
311 + : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
312 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
313 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
314 + "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
315 + : "r"(((unsigned*)(A_shared_warp + 0))[0]),
316 + "r"(((unsigned*)(A_shared_warp + 0))[1]),
317 + "r"(((unsigned*)(A_shared_warp + 0))[2]),
318 + "r"(((unsigned*)(A_shared_warp + 0))[3]),
319 + "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[0]),
320 + "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[1]),
321 + "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
322 + "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
323 + "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
324 + "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
325 }
326
327 {
328 __asm__ __volatile__(
250 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
251 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
252 - : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
253 - : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
329 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
330 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, "
331 + "%13};\n"
332 + : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
333 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
334 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
335 + "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
336 + : "r"(((unsigned*)(A_shared_warp + 0))[0]),
337 + "r"(((unsigned*)(A_shared_warp + 0))[1]),
338 + "r"(((unsigned*)(A_shared_warp + 0))[2]),
339 + "r"(((unsigned*)(A_shared_warp + 0))[3]),
340 + "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]),
341 + "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]),
342 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
343 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
344 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
345 + "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
346 }
347
256 -#endif
348 + #endif
349 }
350 }
351 }
352
261 -// TODO: Shang: Hoist loop invariance.
353 + // TODO: Shang: Hoist loop invariance.
354 for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
355 for (int local_id = 0; local_id < 8; ++local_id) {
264 - int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
265 - if (row_offset < M)
266 - {
267 - *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
356 + int row_offset = (((int)blockIdx_y) / j_factors1) * 16 +
357 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
358 + if (row_offset < M) {
359 + *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 +
360 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
361 }
362 }
363 }
364 #endif
365 }
366
274 -__global__ void __launch_bounds__(64) dequantize_weights(
275 - int* __restrict__ B,
276 - half* __restrict__ scaling_factors,
277 - int* __restrict__ zeros,
278 - half* __restrict__ C,
279 - int G
280 -)
281 -{
367 +__global__ void __launch_bounds__(64)
368 + dequantize_weights(int* __restrict__ B, half* __restrict__ scaling_factors,
369 + int* __restrict__ zeros, half* __restrict__ C, int G) {
370 int j_factors1 = 4;
371 int row_stride2 = 4;
372 int split_k_iters = 1;
@@ -310,14 +398,30 @@ __global__ void __launch_bounds__(64) dequantize_weights(
398
399 uint32_t B_loaded = *(uint32_t*)B_ptr2;
400 uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
313 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
314 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
315 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
316 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
317 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
318 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
319 - asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
320 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
401 + asm volatile("sub.f16x2 %0, %1, %2;\n"
402 + : "=r"(B_loaded_fp16.x)
403 + : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
404 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
405 + : "=r"(B_loaded_fp16.x)
406 + : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
407 + asm volatile("sub.f16x2 %0, %1, %2;\n"
408 + : "=r"(B_loaded_fp16.y)
409 + : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
410 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
411 + : "=r"(B_loaded_fp16.y)
412 + : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
413 + asm volatile("sub.f16x2 %0, %1, %2;\n"
414 + : "=r"(B_loaded_fp16.z)
415 + : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
416 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
417 + : "=r"(B_loaded_fp16.z)
418 + : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
419 + asm volatile("sub.f16x2 %0, %1, %2;\n"
420 + : "=r"(B_loaded_fp16.w)
421 + : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
422 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
423 + : "=r"(B_loaded_fp16.w)
424 + : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
425
426 *(uint4*)B_shared_ptr2 = B_loaded_fp16;
427
@@ -326,58 +430,57 @@ __global__ void __launch_bounds__(64) dequantize_weights(
430 }
431 }
432
329 -} // namespace awq
330 -} // namespace vllm
331 -
332 -torch::Tensor awq_dequantize(
333 - torch::Tensor _kernel,
334 - torch::Tensor _scaling_factors,
335 - torch::Tensor _zeros,
336 - int split_k_iters,
337 - int thx,
338 - int thy)
339 -{
340 - int in_c = _kernel.size(0);
341 - int qout_c = _kernel.size(1);
342 - int out_c = qout_c * 8;
343 - int G = in_c / _scaling_factors.size(0);
344 -
345 - int x_thread = thx;
346 - int y_thread = thy;
347 -
348 - int x_blocks = 1;
349 - int y_blocks = 1;
350 - if (thx==0) {
351 - x_thread = qout_c;
352 - }
353 - if (thy==0) {
354 - y_thread = in_c;
355 - }
356 - if (thx==0 && thy==0) {
357 - x_thread = 8;
358 - y_thread = 8;
359 - x_blocks = (int)(qout_c / 8);
360 - y_blocks = (int)(in_c / 8);
361 - }
433 +} // namespace awq
434 +} // namespace vllm
435 +
436 +torch::Tensor awq_dequantize(torch::Tensor _kernel,
437 + torch::Tensor _scaling_factors,
438 + torch::Tensor _zeros, int split_k_iters, int thx,
439 + int thy) {
440 + int in_c = _kernel.size(0);
441 + int qout_c = _kernel.size(1);
442 + int out_c = qout_c * 8;
443 + int G = in_c / _scaling_factors.size(0);
444 +
445 + int x_thread = thx;
446 + int y_thread = thy;
447 +
448 + int x_blocks = 1;
449 + int y_blocks = 1;
450 + if (thx == 0) {
451 + x_thread = qout_c;
452 + }
453 + if (thy == 0) {
454 + y_thread = in_c;
455 + }
456 + if (thx == 0 && thy == 0) {
457 + x_thread = 8;
458 + y_thread = 8;
459 + x_blocks = (int)(qout_c / 8);
460 + y_blocks = (int)(in_c / 8);
461 + }
462
363 - const at::cuda::OptionalCUDAGuard device_guard(device_of(_scaling_factors));
463 + const at::cuda::OptionalCUDAGuard device_guard(device_of(_scaling_factors));
464
365 - auto options = torch::TensorOptions().dtype(_scaling_factors.dtype()).device(_scaling_factors.device());
366 - at::Tensor _de_kernel = torch::empty({in_c, out_c}, options);
465 + auto options = torch::TensorOptions()
466 + .dtype(_scaling_factors.dtype())
467 + .device(_scaling_factors.device());
468 + at::Tensor _de_kernel = torch::empty({in_c, out_c}, options);
469
368 - auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
369 - auto de_kernel = reinterpret_cast<half*>(_de_kernel.data_ptr<at::Half>());
370 - auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
371 - auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
470 + auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
471 + auto de_kernel = reinterpret_cast<half*>(_de_kernel.data_ptr<at::Half>());
472 + auto scaling_factors =
473 + reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
474 + auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
475
373 - dim3 num_blocks(x_blocks, y_blocks);
374 - dim3 threads_per_block(x_thread, y_thread);
476 + dim3 num_blocks(x_blocks, y_blocks);
477 + dim3 threads_per_block(x_thread, y_thread);
478
376 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
377 - vllm::awq::dequantize_weights<<<num_blocks, threads_per_block, 0, stream>>>(
378 - kernel, scaling_factors, zeros, de_kernel, G);
479 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
480 + vllm::awq::dequantize_weights<<<num_blocks, threads_per_block, 0, stream>>>(
481 + kernel, scaling_factors, zeros, de_kernel, G);
482
380 - return _de_kernel;
483 + return _de_kernel;
484 }
485
486 // in_feats: M, IC [float16]
@@ -386,61 +489,61 @@ torch::Tensor awq_dequantize(
489 // zeros: IC // G, OC // 8 [int32] -> cast to IC // G, OC [uint4b]
490 // assume that batch_size < 16 for now
491
389 -torch::Tensor awq_gemm(
390 - torch::Tensor _in_feats,
391 - torch::Tensor _kernel,
392 - torch::Tensor _scaling_factors,
393 - torch::Tensor _zeros,
394 - int split_k_iters)
395 -{
396 - int num_in_feats = _in_feats.size(0);
397 - int num_in_channels = _in_feats.size(1);
398 - const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
399 -
400 - auto options = torch::TensorOptions().dtype(_in_feats.dtype()).device(_in_feats.device());
401 - at::Tensor _out_feats = torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
402 - int num_out_feats = _out_feats.size(-2);
403 - int num_out_channels = _out_feats.size(-1);
404 -
405 - auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
406 - auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
407 - auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
408 - auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
409 - auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
410 - int group_size = num_in_channels / _scaling_factors.size(0);
411 -
412 - if (num_out_channels % 64 != 0)
413 - throw std::invalid_argument("OC is not multiple of cta_N = 64");
414 - if (num_out_channels % 8 != 0)
415 - throw std::invalid_argument("OC is not multiple of pack_num = 8");
416 - if (group_size % 32 != 0)
417 - throw std::invalid_argument("Group size should be a multiple of 32");
418 - if (num_out_channels % group_size != 0)
419 - throw std::invalid_argument("OC is not multiple of Group size");
420 -
421 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
422 - if (num_out_channels % 128 == 0)
423 - {
424 - int j_factors1 = num_out_channels / 128 / 1;
425 - dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
426 - // threadIdx.x: 32
427 - // threadIdx.y: i_factors[2] * j_factors[2]
428 - dim3 threads_per_block(32, 2);
429 - vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128><<<num_blocks, threads_per_block, 0, stream>>>(
430 - group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
431 - num_out_channels, out_feats);
432 - }
433 - else if (num_out_channels % 64 == 0)
434 - {
435 - int j_factors1 = num_out_channels / 64 / 1;
436 - dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
437 -
438 - // threadIdx.x: 32
439 - // threadIdx.y: i_factors[2] * j_factors[2]
440 - dim3 threads_per_block(32, 2);
441 - vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64><<<num_blocks, threads_per_block, 0, stream>>>(
442 - group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
443 - num_out_channels, out_feats);
444 - }
445 - return _out_feats.sum(0);
492 +torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
493 + torch::Tensor _scaling_factors, torch::Tensor _zeros,
494 + int split_k_iters) {
495 + int num_in_feats = _in_feats.size(0);
496 + int num_in_channels = _in_feats.size(1);
497 + const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
498 +
499 + auto options = torch::TensorOptions()
500 + .dtype(_in_feats.dtype())
501 + .device(_in_feats.device());
502 + at::Tensor _out_feats =
503 + torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
504 + int num_out_feats = _out_feats.size(-2);
505 + int num_out_channels = _out_feats.size(-1);
506 +
507 + auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
508 + auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
509 + auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
510 + auto scaling_factors =
511 + reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
512 + auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
513 + int group_size = num_in_channels / _scaling_factors.size(0);
514 +
515 + if (num_out_channels % 64 != 0)
516 + throw std::invalid_argument("OC is not multiple of cta_N = 64");
517 + if (num_out_channels % 8 != 0)
518 + throw std::invalid_argument("OC is not multiple of pack_num = 8");
519 + if (group_size % 32 != 0)
520 + throw std::invalid_argument("Group size should be a multiple of 32");
521 + if (num_out_channels % group_size != 0)
522 + throw std::invalid_argument("OC is not multiple of Group size");
523 +
524 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
525 + if (num_out_channels % 128 == 0) {
526 + int j_factors1 = num_out_channels / 128 / 1;
527 + dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
528 + // threadIdx.x: 32
529 + // threadIdx.y: i_factors[2] * j_factors[2]
530 + dim3 threads_per_block(32, 2);
531 + vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128>
532 + <<<num_blocks, threads_per_block, 0, stream>>>(
533 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros,
534 + num_in_feats, num_in_channels, num_out_channels, out_feats);
535 + } else if (num_out_channels % 64 == 0) {
536 + int j_factors1 = num_out_channels / 64 / 1;
537 + dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 *
538 + split_k_iters);
539 +
540 + // threadIdx.x: 32
541 + // threadIdx.y: i_factors[2] * j_factors[2]
542 + dim3 threads_per_block(32, 2);
543 + vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64>
544 + <<<num_blocks, threads_per_block, 0, stream>>>(
545 + group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros,
546 + num_in_feats, num_in_channels, num_out_channels, out_feats);
547 + }
548 + return _out_feats.sum(0);
549 }
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+19 -19
@@ -117,10 +117,10 @@ struct cutlass_2x_gemm {
117 };
118
119 template <typename Gemm>
120 -void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
121 - torch::Tensor const &b,
122 - torch::Tensor const &a_scales,
123 - torch::Tensor const &b_scales) {
120 +void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
121 + torch::Tensor const& b,
122 + torch::Tensor const& a_scales,
123 + torch::Tensor const& b_scales) {
124 using ElementAB = typename Gemm::ElementAB;
125 using ElementD = typename Gemm::ElementD;
126
@@ -136,9 +136,9 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
136 using StrideC = Stride<int64_t, Int<1>, Int<0>>;
137 StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
138
139 - auto a_ptr = static_cast<ElementAB const *>(a.data_ptr());
140 - auto b_ptr = static_cast<ElementAB const *>(b.data_ptr());
141 - auto c_ptr = static_cast<ElementD *>(out.data_ptr());
139 + auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
140 + auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
141 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
142
143 auto a_scales_ptr = a_scales.data_ptr<float>();
144 auto b_scales_ptr = b_scales.data_ptr<float>();
@@ -196,10 +196,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
196
197 } // namespace
198
199 -void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
200 - torch::Tensor const &b,
201 - torch::Tensor const &a_scales,
202 - torch::Tensor const &b_scales) {
199 +void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
200 + torch::Tensor const& b,
201 + torch::Tensor const& a_scales,
202 + torch::Tensor const& b_scales) {
203 TORCH_CHECK(a.dtype() == torch::kInt8);
204 TORCH_CHECK(b.dtype() == torch::kInt8);
205 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -223,10 +223,10 @@ void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
223 }
224 }
225
226 -void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
227 - torch::Tensor const &b,
228 - torch::Tensor const &a_scales,
229 - torch::Tensor const &b_scales) {
226 +void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
227 + torch::Tensor const& b,
228 + torch::Tensor const& a_scales,
229 + torch::Tensor const& b_scales) {
230 TORCH_CHECK(a.dtype() == torch::kInt8);
231 TORCH_CHECK(b.dtype() == torch::kInt8);
232 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -250,10 +250,10 @@ void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
250 }
251 }
252
253 -void cutlass_scaled_mm_dq_sm89(torch::Tensor &out, torch::Tensor const &a,
254 - torch::Tensor const &b,
255 - torch::Tensor const &a_scales,
256 - torch::Tensor const &b_scales) {
253 +void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
254 + torch::Tensor const& b,
255 + torch::Tensor const& a_scales,
256 + torch::Tensor const& b_scales) {
257 using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
258 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
259 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+11 -11
@@ -120,10 +120,10 @@ struct cutlass_3x_gemm {
120 };
121
122 template <typename Gemm>
123 -void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
124 - torch::Tensor const &b,
125 - torch::Tensor const &a_scales,
126 - torch::Tensor const &b_scales) {
123 +void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
124 + torch::Tensor const& b,
125 + torch::Tensor const& a_scales,
126 + torch::Tensor const& b_scales) {
127 using ElementAB = typename Gemm::ElementAB;
128 using ElementD = typename Gemm::ElementD;
129
@@ -146,12 +146,12 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
146 using GemmKernel = typename Gemm::GemmKernel;
147 typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
148
149 - auto a_ptr = static_cast<ElementAB *>(a.data_ptr());
150 - auto b_ptr = static_cast<ElementAB *>(b.data_ptr());
149 + auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
150 + auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
151 typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
152 b_stride};
153
154 - auto c_ptr = static_cast<ElementD *>(out.data_ptr());
154 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
155 typename GemmKernel::EpilogueArguments epilogue_args{
156 {}, c_ptr, c_stride, c_ptr, c_stride};
157
@@ -183,10 +183,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
183 }
184 } // namespace
185
186 -void cutlass_scaled_mm_dq_sm90(torch::Tensor &out, torch::Tensor const &a,
187 - torch::Tensor const &b,
188 - torch::Tensor const &a_scales,
189 - torch::Tensor const &b_scales) {
186 +void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
187 + torch::Tensor const& b,
188 + torch::Tensor const& a_scales,
189 + torch::Tensor const& b_scales) {
190 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
191 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
192
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+24 -23
@@ -2,29 +2,29 @@
2 #include <cuda_runtime.h>
3 #include <torch/extension.h>
4
5 -void cutlass_scaled_mm_dq_sm75(torch::Tensor &c, torch::Tensor const &a,
6 - torch::Tensor const &b,
7 - torch::Tensor const &a_scales,
8 - torch::Tensor const &b_scales);
5 +void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
6 + torch::Tensor const& b,
7 + torch::Tensor const& a_scales,
8 + torch::Tensor const& b_scales);
9
10 -void cutlass_scaled_mm_dq_sm80(torch::Tensor &c, torch::Tensor const &a,
11 - torch::Tensor const &b,
12 - torch::Tensor const &a_scales,
13 - torch::Tensor const &b_scales);
10 +void cutlass_scaled_mm_dq_sm80(torch::Tensor& c, torch::Tensor const& a,
11 + torch::Tensor const& b,
12 + torch::Tensor const& a_scales,
13 + torch::Tensor const& b_scales);
14
15 -void cutlass_scaled_mm_dq_sm89(torch::Tensor &c, torch::Tensor const &a,
16 - torch::Tensor const &b,
17 - torch::Tensor const &a_scales,
18 - torch::Tensor const &b_scales);
15 +void cutlass_scaled_mm_dq_sm89(torch::Tensor& c, torch::Tensor const& a,
16 + torch::Tensor const& b,
17 + torch::Tensor const& a_scales,
18 + torch::Tensor const& b_scales);
19
20 -void cutlass_scaled_mm_dq_sm90(torch::Tensor &c, torch::Tensor const &a,
21 - torch::Tensor const &b,
22 - torch::Tensor const &a_scales,
23 - torch::Tensor const &b_scales);
20 +void cutlass_scaled_mm_dq_sm90(torch::Tensor& c, torch::Tensor const& a,
21 + torch::Tensor const& b,
22 + torch::Tensor const& a_scales,
23 + torch::Tensor const& b_scales);
24
25 -void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
26 - torch::Tensor const &b, torch::Tensor const &a_scales,
27 - torch::Tensor const &b_scales) {
25 +void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
26 + torch::Tensor const& b, torch::Tensor const& a_scales,
27 + torch::Tensor const& b_scales) {
28 int32_t major_capability;
29 int32_t minor_capability;
30 cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
@@ -36,14 +36,15 @@ void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
36 // Checks for conformality
37 TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
38 TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
39 - b.size(1) == c.size(1));
39 + b.size(1) == c.size(1));
40 TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
41 TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
42
43 // Check for strides and alignment
44 - TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45 - TORCH_CHECK(b.stride(0) == 1); // Column-major
46 - TORCH_CHECK(c.stride(0) % 16 == 0 && b.stride(1) % 16 == 0); // 16 Byte Alignment
44 + TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45 + TORCH_CHECK(b.stride(0) == 1); // Column-major
46 + TORCH_CHECK(c.stride(0) % 16 == 0 &&
47 + b.stride(1) % 16 == 0); // 16 Byte Alignment
48 TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
49
50 at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
csrc/quantization/fp8/amd/hip_float8.h
+93 -123
@@ -1,167 +1,137 @@
1 #pragma once
2
3 #ifdef __HIPCC__
4 -#include <hip/hip_runtime.h>
4 + #include <hip/hip_runtime.h>
5 #else
6 -#include <type_traits>
7 -#include <stdint.h>
8 -#include <math.h>
9 -#include <iostream>
6 + #include <type_traits>
7 + #include <stdint.h>
8 + #include <math.h>
9 + #include <iostream>
10 #endif
11
12 #include "hip_float8_impl.h"
13
14 -struct alignas(1) hip_fp8
15 -{
16 - struct from_bits_t
17 - {
18 - };
19 - HIP_FP8_HOST_DEVICE static constexpr from_bits_t from_bits() { return from_bits_t(); }
20 - uint8_t data;
21 -
22 - hip_fp8() = default;
23 - HIP_FP8_HOST_DEVICE constexpr hip_fp8(const hip_fp8&) = default;
24 - HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v) = delete;
25 - explicit HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v, from_bits_t)
26 - : data(v)
27 - {
28 - }
14 +struct alignas(1) hip_fp8 {
15 + struct from_bits_t {};
16 + HIP_FP8_HOST_DEVICE static constexpr from_bits_t from_bits() {
17 + return from_bits_t();
18 + }
19 + uint8_t data;
20 +
21 + hip_fp8() = default;
22 + HIP_FP8_HOST_DEVICE constexpr hip_fp8(const hip_fp8&) = default;
23 + HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v) = delete;
24 + explicit HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v, from_bits_t)
25 + : data(v) {}
26
27 #ifdef __HIP__MI300__
31 - // NOTE: ON-DEVICE... always optimal bias
32 - explicit HIP_FP8_DEVICE hip_fp8(float v)
33 - : data(hip_fp8_impl::to_fp8_from_fp32(v))
34 - {
35 - }
36 -
37 - explicit HIP_FP8_DEVICE hip_fp8(_Float16 v)
38 - : hip_fp8(static_cast<float>(v))
39 - {
40 - }
41 -
42 - // Host only implementation using s/w simulation
43 - explicit HIP_FP8_HOST
44 -#else // __HIP__MI300__
45 - // both Host and DEVICE for non-MI300 using s/w simulation
46 - explicit HIP_FP8_HOST_DEVICE
47 -#endif // __HIP__MI300__
48 - hip_fp8(float v)
49 - {
50 - data = hip_fp8_impl::to_float8<4, 3, float, true /*negative_zero_nan*/, true /*clip*/>(v);
51 - }
52 -
53 - explicit HIP_FP8_HOST_DEVICE hip_fp8(double v)
54 - : hip_fp8(static_cast<float>(v))
55 - {
56 - }
28 + // NOTE: ON-DEVICE... always optimal bias
29 + explicit HIP_FP8_DEVICE hip_fp8(float v)
30 + : data(hip_fp8_impl::to_fp8_from_fp32(v)) {}
31 +
32 + explicit HIP_FP8_DEVICE hip_fp8(_Float16 v)
33 + : hip_fp8(static_cast<float>(v)) {}
34 +
35 + // Host only implementation using s/w simulation
36 + explicit HIP_FP8_HOST
37 +#else // __HIP__MI300__
38 + // both Host and DEVICE for non-MI300 using s/w simulation
39 + explicit HIP_FP8_HOST_DEVICE
40 +#endif // __HIP__MI300__
41 + hip_fp8(float v) {
42 + data = hip_fp8_impl::to_float8<4, 3, float, true /*negative_zero_nan*/,
43 + true /*clip*/>(v);
44 + }
45 +
46 + explicit HIP_FP8_HOST_DEVICE hip_fp8(double v)
47 + : hip_fp8(static_cast<float>(v)) {}
48
49 #ifdef __HIP__MI300__
59 - // upcast using device specific intrinsic
60 - explicit inline HIP_FP8_DEVICE operator float() const
61 - {
62 - float fval;
63 - uint32_t i32val = static_cast<uint32_t>(data);
64 -
65 - // upcast
66 - asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val));
67 -
68 - return fval;
69 - }
70 -
71 - explicit inline HIP_FP8_HOST operator float() const
72 -#else // __HIP__MI300__
73 - explicit inline HIP_FP8_HOST_DEVICE operator float() const
74 -#endif // __HIP__MI300__
75 - {
76 - return hip_fp8_impl::from_float8<4, 3, float, true /*negative_zero_nan*/>(data);
77 - }
50 + // upcast using device specific intrinsic
51 + explicit inline HIP_FP8_DEVICE operator float() const {
52 + float fval;
53 + uint32_t i32val = static_cast<uint32_t>(data);
54 +
55 + // upcast
56 + asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0"
57 + : "=v"(fval)
58 + : "v"(i32val));
59 +
60 + return fval;
61 + }
62 +
63 + explicit inline HIP_FP8_HOST operator float() const
64 +#else // __HIP__MI300__
65 + explicit inline HIP_FP8_HOST_DEVICE operator float() const
66 +#endif // __HIP__MI300__
67 + {
68 + return hip_fp8_impl::from_float8<4, 3, float, true /*negative_zero_nan*/>(
69 + data);
70 + }
71 };
72
80 -namespace std
81 -{
82 -inline hip_fp8 sin(hip_fp8 a)
83 -{
84 - return hip_fp8(sinf(float(a)));
85 -}
86 -inline hip_fp8 cos(hip_fp8 a)
87 -{
88 - return hip_fp8(cosf(float(a)));
89 -}
90 -HIP_FP8_HOST_DEVICE constexpr hip_fp8 real(const hip_fp8& a)
91 -{
92 - return a;
93 -}
94 -} // namespace std
73 +namespace std {
74 +inline hip_fp8 sin(hip_fp8 a) { return hip_fp8(sinf(float(a))); }
75 +inline hip_fp8 cos(hip_fp8 a) { return hip_fp8(cosf(float(a))); }
76 +HIP_FP8_HOST_DEVICE constexpr hip_fp8 real(const hip_fp8& a) { return a; }
77 +} // namespace std
78
79 // Special operator overloading
97 -inline std::ostream& operator<<(std::ostream& os, const hip_fp8& f8)
98 -{
99 - return os << float(f8);
80 +inline std::ostream& operator<<(std::ostream& os, const hip_fp8& f8) {
81 + return os << float(f8);
82 }
83
84 // all + operator overloading with mixed types
103 -// mixed types, always converts to f32, does computation in f32, and returns float
104 -inline HIP_FP8_HOST_DEVICE float operator+(const float fa, hip_fp8 b)
105 -{
106 - return (fa + float(b));
85 +// mixed types, always converts to f32, does computation in f32, and returns
86 +// float
87 +inline HIP_FP8_HOST_DEVICE float operator+(const float fa, hip_fp8 b) {
88 + return (fa + float(b));
89 }
90
109 -inline HIP_FP8_HOST_DEVICE float operator+(hip_fp8 a, const float fb)
110 -{
111 - return (float(a) + fb);
91 +inline HIP_FP8_HOST_DEVICE float operator+(hip_fp8 a, const float fb) {
92 + return (float(a) + fb);
93 }
94
114 -inline HIP_FP8_HOST_DEVICE hip_fp8 operator+(hip_fp8 a, hip_fp8 b)
115 -{
116 - return hip_fp8(float(a) + float(b));
95 +inline HIP_FP8_HOST_DEVICE hip_fp8 operator+(hip_fp8 a, hip_fp8 b) {
96 + return hip_fp8(float(a) + float(b));
97 }
98
119 -inline HIP_FP8_HOST_DEVICE hip_fp8& operator+=(hip_fp8& a, hip_fp8 b)
120 -{
121 - return a = hip_fp8(float(a) + float(b));
99 +inline HIP_FP8_HOST_DEVICE hip_fp8& operator+=(hip_fp8& a, hip_fp8 b) {
100 + return a = hip_fp8(float(a) + float(b));
101 }
102
103 // overloading multiplication, always returns float,
125 -inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, hip_fp8 b)
126 -{
127 - return float(a) * float(b);
104 +inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, hip_fp8 b) {
105 + return float(a) * float(b);
106 }
107
130 -inline HIP_FP8_HOST_DEVICE float operator*(float a, hip_fp8 b)
131 -{
132 - return (a * float(b));
108 +inline HIP_FP8_HOST_DEVICE float operator*(float a, hip_fp8 b) {
109 + return (a * float(b));
110 }
111
135 -inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, float b)
136 -{
137 - return (float(a) * b);
112 +inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, float b) {
113 + return (float(a) * b);
114 }
115
140 -inline HIP_FP8_HOST_DEVICE float operator*(int32_t a, hip_fp8 b)
141 -{
142 - return ((float)a * float(b));
116 +inline HIP_FP8_HOST_DEVICE float operator*(int32_t a, hip_fp8 b) {
117 + return ((float)a * float(b));
118 }
119
145 -inline HIP_FP8_HOST_DEVICE float operator*(double a, hip_fp8 b)
146 -{
147 - return ((float)a * float(b));
120 +inline HIP_FP8_HOST_DEVICE float operator*(double a, hip_fp8 b) {
121 + return ((float)a * float(b));
122 }
123
124 // overloading for compare
151 -inline HIP_FP8_HOST_DEVICE bool operator==(hip_fp8 a, hip_fp8 b)
152 -{
153 - return (a.data == b.data);
125 +inline HIP_FP8_HOST_DEVICE bool operator==(hip_fp8 a, hip_fp8 b) {
126 + return (a.data == b.data);
127 }
155 -inline HIP_FP8_HOST_DEVICE bool operator!=(hip_fp8 a, hip_fp8 b)
156 -{
157 - return (a.data != b.data);
128 +inline HIP_FP8_HOST_DEVICE bool operator!=(hip_fp8 a, hip_fp8 b) {
129 + return (a.data != b.data);
130 }
131
160 -inline HIP_FP8_HOST_DEVICE bool operator>=(hip_fp8 a, hip_fp8 b)
161 -{
162 - return static_cast<float>(a) >= static_cast<float>(b);
132 +inline HIP_FP8_HOST_DEVICE bool operator>=(hip_fp8 a, hip_fp8 b) {
133 + return static_cast<float>(a) >= static_cast<float>(b);
134 }
164 -inline HIP_FP8_HOST_DEVICE bool operator>(hip_fp8 a, hip_fp8 b)
165 -{
166 - return static_cast<float>(a) > static_cast<float>(b);
135 +inline HIP_FP8_HOST_DEVICE bool operator>(hip_fp8 a, hip_fp8 b) {
136 + return static_cast<float>(a) > static_cast<float>(b);
137 }
csrc/quantization/fp8/amd/hip_float8_impl.h
+260 -260
@@ -1,316 +1,316 @@
1 #pragma once
2
3 -#if defined(__HIPCC__) && (defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__))
4 -#define __HIP__MI300__
3 +#if defined(__HIPCC__) && \
4 + (defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__))
5 + #define __HIP__MI300__
6 #endif
7
8 #ifdef __HIPCC__
8 -#define HIP_FP8_HOST_DEVICE __host__ __device__
9 -#define HIP_FP8_HOST __host__
10 -#define HIP_FP8_DEVICE __device__
9 + #define HIP_FP8_HOST_DEVICE __host__ __device__
10 + #define HIP_FP8_HOST __host__
11 + #define HIP_FP8_DEVICE __device__
12 #else
12 -#define HIP_FP8_HOST_DEVICE
13 -#define HIP_FP8_HOST
14 -#define HIP_FP8_DEVICE
13 + #define HIP_FP8_HOST_DEVICE
14 + #define HIP_FP8_HOST
15 + #define HIP_FP8_DEVICE
16 #endif
17
17 -namespace hip_fp8_impl
18 -{
18 +namespace hip_fp8_impl {
19
20 #ifdef __HIP__MI300__
21 -HIP_FP8_DEVICE uint8_t to_fp8_from_fp32(float v)
22 -{
23 - uint8_t i8data;
24 - union {
25 - float fval;
26 - uint32_t i32val;
27 - uint8_t i8val[4]; // NOTE: not endian independent
28 - } val;
29 -
30 - uint32_t ival = 0;
31 - val.fval = v;
32 -
33 - if ((val.i32val & 0x7F800000) != 0x7F800000) { /// propagate NAN/INF, no clipping
34 - val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0);
35 - }
36 -
37 - ival = __builtin_amdgcn_cvt_pk_fp8_f32(val.fval, val.fval, ival,
38 - false); // false -> WORD0
39 - val.i32val = ival;
40 - i8data = val.i8val[0];
41 -
42 - return i8data;
21 +HIP_FP8_DEVICE uint8_t to_fp8_from_fp32(float v) {
22 + uint8_t i8data;
23 + union {
24 + float fval;
25 + uint32_t i32val;
26 + uint8_t i8val[4]; // NOTE: not endian independent
27 + } val;
28 +
29 + uint32_t ival = 0;
30 + val.fval = v;
31 +
32 + if ((val.i32val & 0x7F800000) !=
33 + 0x7F800000) { /// propagate NAN/INF, no clipping
34 + val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0);
35 + }
36 +
37 + ival = __builtin_amdgcn_cvt_pk_fp8_f32(val.fval, val.fval, ival,
38 + false); // false -> WORD0
39 + val.i32val = ival;
40 + i8data = val.i8val[0];
41 +
42 + return i8data;
43 }
44 -#endif // __HIP__MI300__
44 +#endif // __HIP__MI300__
45
46 -HIP_FP8_HOST inline int clz(uint32_t x)
47 -{
48 - return __builtin_clz(x);
49 -}
46 +HIP_FP8_HOST inline int clz(uint32_t x) { return __builtin_clz(x); }
47 #if defined(__HIPCC__) || defined(__CUDA_ARCH__)
51 -HIP_FP8_DEVICE inline int clz(uint32_t x)
52 -{
53 - return __clz(x);
54 -}
48 +HIP_FP8_DEVICE inline int clz(uint32_t x) { return __clz(x); }
49 #endif
50
51 template <int we, int wm, typename T, bool negative_zero_nan, bool clip>
58 -HIP_FP8_HOST_DEVICE uint8_t to_float8(T _x, bool stoch = false, uint32_t rng = 0)
59 -{
52 +HIP_FP8_HOST_DEVICE uint8_t to_float8(T _x, bool stoch = false,
53 + uint32_t rng = 0) {
54 #ifdef __HIPCC__
61 - constexpr bool is_half = std::is_same<T, _Float16>::value;
55 + constexpr bool is_half = std::is_same<T, _Float16>::value;
56 #else
63 - constexpr bool is_half = false;
57 + constexpr bool is_half = false;
58 #endif
65 - constexpr bool is_float = std::is_same<T, float>::value;
66 - static_assert(wm + we == 7, "wm+we==7");
67 - static_assert(is_half || is_float, "Only half and float can be cast to f8");
68 -
69 - const int mfmt = (sizeof(T) == 4) ? 23 : 10;
70 - uint32_t x;
59 + constexpr bool is_float = std::is_same<T, float>::value;
60 + static_assert(wm + we == 7, "wm+we==7");
61 + static_assert(is_half || is_float, "Only half and float can be cast to f8");
62 +
63 + const int mfmt = (sizeof(T) == 4) ? 23 : 10;
64 + uint32_t x;
65 + if (sizeof(T) == 4) {
66 + x = reinterpret_cast<uint32_t&>(_x);
67 + } else {
68 + x = reinterpret_cast<uint16_t&>(_x);
69 + }
70 +
71 + uint32_t head, mantissa;
72 + int exponent, bias;
73 + uint32_t sign;
74 +
75 + if (sizeof(T) == 4) {
76 + head = x & 0xFF800000;
77 + mantissa = x & 0x7FFFFF;
78 + exponent = (head >> 23) & 0xFF;
79 + sign = head >> 31;
80 + bias = 127;
81 + } else {
82 + head = x & 0xFC00;
83 + mantissa = x & 0x3FF;
84 + exponent = (head >> 10) & 0x1F;
85 + sign = head >> 15;
86 + bias = 15;
87 + }
88 +
89 + uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm);
90 +
91 + // Deal with inf and NaNs
92 + if (negative_zero_nan) {
93 if (sizeof(T) == 4) {
72 - x = reinterpret_cast<uint32_t&>(_x);
94 + if ((x & 0x7F800000) == 0x7F800000) {
95 + return 0x80;
96 + }
97 } else {
74 - x = reinterpret_cast<uint16_t&>(_x);
98 + // if(__hisinf(x) || __hisnan(x))
99 + if ((x & 0x7C00) == 0x7C00) {
100 + return 0x80;
101 + }
102 }
76 -
77 - uint32_t head, mantissa;
78 - int exponent, bias;
79 - uint32_t sign;
80 -
103 + } else {
104 if (sizeof(T) == 4) {
82 - head = x & 0xFF800000;
83 - mantissa = x & 0x7FFFFF;
84 - exponent = (head >> 23) & 0xFF;
85 - sign = head >> 31;
86 - bias = 127;
105 + if ((x & 0x7F800000) == 0x7F800000) {
106 + return signed_inf + (mantissa != 0 ? 1 : 0);
107 + }
108 } else {
88 - head = x & 0xFC00;
89 - mantissa = x & 0x3FF;
90 - exponent = (head >> 10) & 0x1F;
91 - sign = head >> 15;
92 - bias = 15;
109 + if ((x & 0x7C00) == 0x7C00) {
110 + return signed_inf + (mantissa != 0 ? 1 : 0);
111 + }
112 }
94 -
95 - uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm);
96 -
97 - // Deal with inf and NaNs
98 - if (negative_zero_nan) {
99 - if (sizeof(T) == 4) {
100 - if ((x & 0x7F800000) == 0x7F800000) {
101 - return 0x80;
102 - }
103 - } else {
104 - // if(__hisinf(x) || __hisnan(x))
105 - if ((x & 0x7C00) == 0x7C00) {
106 - return 0x80;
107 - }
108 - }
109 - } else {
110 - if (sizeof(T) == 4) {
111 - if ((x & 0x7F800000) == 0x7F800000) {
112 - return signed_inf + (mantissa != 0 ? 1 : 0);
113 - }
114 - } else {
115 - if ((x & 0x7C00) == 0x7C00) {
116 - return signed_inf + (mantissa != 0 ? 1 : 0);
117 - }
118 - }
119 - }
120 - if (x == 0) {
121 - return 0;
122 - }
123 -
124 - // First need to check if it is normal or denorm as there is a difference of
125 - // implicit 1 Then need to adjust the exponent to align with the F8 exponent,
126 - // in the meanwhile, shift The mantissa. Then for stochastic rounding, add rng
127 - // to mantissa and truncate. And for RNE, no need to add rng. Then probably
128 - // need to check whether there is carry and adjust exponent and mantissa again
129 -
130 - // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent
131 - // bits
132 - const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0);
133 - const int f8_denormal_act_exponent = 1 - f8_bias; // actual exponent of f8 denormal
134 - // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias)
135 - // f8_exponent is the converted f8 exponent with bias encoding
136 - // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent,
137 - // the difference needs to be adjusted and mantissa shifted
138 - int act_exponent, f8_exponent, exponent_diff;
139 -
140 - if (exponent == 0) { // fp32/fp16 is in denormal.
141 - /* fp32 denormal is below 2^-127 so it is usually not a concern here, we
113 + }
114 + if (x == 0) {
115 + return 0;
116 + }
117 +
118 + // First need to check if it is normal or denorm as there is a difference of
119 + // implicit 1 Then need to adjust the exponent to align with the F8 exponent,
120 + // in the meanwhile, shift The mantissa. Then for stochastic rounding, add rng
121 + // to mantissa and truncate. And for RNE, no need to add rng. Then probably
122 + // need to check whether there is carry and adjust exponent and mantissa again
123 +
124 + // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent
125 + // bits
126 + const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0);
127 + const int f8_denormal_act_exponent =
128 + 1 - f8_bias; // actual exponent of f8 denormal
129 + // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias)
130 + // f8_exponent is the converted f8 exponent with bias encoding
131 + // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent,
132 + // the difference needs to be adjusted and mantissa shifted
133 + int act_exponent, f8_exponent, exponent_diff;
134 +
135 + if (exponent == 0) { // fp32/fp16 is in denormal.
136 + /* fp32 denormal is below 2^-127 so it is usually not a concern here, we
137 mostly concern fp16 here. In this case, f8 is usually in denormal. But there
138 could be exceptions. fp16 denormal has exponent bias 15 while bf8 with NANOO has
139 exponent bias 16. It means that there are some numbers in fp16 denormal but they
140 are bf8 (NANOO) normals - smallest bf8 (NANOO) normal is 2^-15. fp16 numbers
141 where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 are bf8
142 (NANOO) normal. In this case, the fp16 mantissa should be shift left by 1 */
148 - act_exponent = exponent - bias + 1;
149 - exponent_diff = f8_denormal_act_exponent - act_exponent; // actual exponent is exponent-bias+1 as it is denormal
150 - } else { // fp32/fp16 is normal with implicit 1
151 - act_exponent = exponent - bias;
152 - if (act_exponent <= f8_denormal_act_exponent) {
153 - /* This is the case where fp32/fp16 is normal but it is in f8 denormal
154 - range. For example fp8 nanoo mode, denormal exponent is -7, but if the
155 - fp32/fp16 actual exponent is -7, it is actually larger due to the implicit 1,
156 - Therefore it needs to be adjust to -6 and mantissa shift right by 1.
157 - So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */
158 - exponent_diff = f8_denormal_act_exponent - act_exponent;
159 - } else { // both fp32/fp16 and f8 are in normal range
160 - exponent_diff = 0; // exponent_diff=0 does not mean there is no difference
161 - // for this case,
162 - // act_exponent could be larger. Just that it does not need shift mantissa
163 - }
164 - mantissa += (1 << mfmt); // Add the implicit 1 into mantissa
143 + act_exponent = exponent - bias + 1;
144 + exponent_diff =
145 + f8_denormal_act_exponent -
146 + act_exponent; // actual exponent is exponent-bias+1 as it is denormal
147 + } else { // fp32/fp16 is normal with implicit 1
148 + act_exponent = exponent - bias;
149 + if (act_exponent <= f8_denormal_act_exponent) {
150 + /* This is the case where fp32/fp16 is normal but it is in f8 denormal
151 +range. For example fp8 nanoo mode, denormal exponent is -7, but if the
152 +fp32/fp16 actual exponent is -7, it is actually larger due to the implicit 1,
153 +Therefore it needs to be adjust to -6 and mantissa shift right by 1.
154 +So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */
155 + exponent_diff = f8_denormal_act_exponent - act_exponent;
156 + } else { // both fp32/fp16 and f8 are in normal range
157 + exponent_diff = 0; // exponent_diff=0 does not mean there is no
158 + // difference for this case, act_exponent could be
159 + // larger. Just that it does not need shift mantissa
160 }
166 -
167 - bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) ==
168 - static_cast<uint32_t>(1 << (mfmt - wm + exponent_diff - 1));
169 - /* This part is a bit tricky. The judgment of whether it is a tie needs to be
170 - done before we shift right as shift right could rip off some residual part
171 - and make something not midpoint look like midpoint. For example, the fp16
172 - number 0x1002 (0 00100 0000000010), it is larger than midpoint, but after
173 - shift right by 4 bits, it would look like midpoint.
161 + mantissa += (1 << mfmt); // Add the implicit 1 into mantissa
162 + }
163 +
164 + bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) ==
165 + static_cast<uint32_t>(1 << (mfmt - wm + exponent_diff - 1));
166 + /* This part is a bit tricky. The judgment of whether it is a tie needs to be
167 + done before we shift right as shift right could rip off some residual part
168 + and make something not midpoint look like midpoint. For example, the fp16
169 + number 0x1002 (0 00100 0000000010), it is larger than midpoint, but after
170 + shift right by 4 bits, it would look like midpoint.
171 */
172
176 - if (exponent_diff > 0) {
177 - mantissa >>= exponent_diff;
178 - } else if (exponent_diff == -1) {
179 - mantissa <<= -exponent_diff;
173 + if (exponent_diff > 0) {
174 + mantissa >>= exponent_diff;
175 + } else if (exponent_diff == -1) {
176 + mantissa <<= -exponent_diff;
177 + }
178 + bool implicit_one = mantissa & (1 << mfmt);
179 + // if there is no implicit 1, it means the f8 is denormal and need to adjust
180 + // to denorm exponent
181 + f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ +
182 + f8_bias - (implicit_one ? 0 : 1);
183 +
184 + // Now we have the exponent and mantissa adjusted
185 + uint32_t drop_mask = (1 << (mfmt - wm)) - 1;
186 + bool odd = mantissa & (1 << (mfmt - wm)); // if the least significant bit
187 + // that is not truncated is 1
188 + mantissa +=
189 + (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) &
190 + drop_mask;
191 +
192 + // Now we deal with overflow
193 + if (f8_exponent == 0) {
194 + if ((1 << mfmt) & mantissa) {
195 + f8_exponent = 1; // denormal overflow to become normal, promote exponent
196 }
181 - bool implicit_one = mantissa & (1 << mfmt);
182 - // if there is no implicit 1, it means the f8 is denormal and need to adjust
183 - // to denorm exponent
184 - f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias - (implicit_one ? 0 : 1);
185 -
186 - // Now we have the exponent and mantissa adjusted
187 - uint32_t drop_mask = (1 << (mfmt - wm)) - 1;
188 - bool odd = mantissa & (1 << (mfmt - wm)); // if the least significant bit that
189 - // is not truncated is 1
190 - mantissa += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask;
191 -
192 - // Now we deal with overflow
193 - if (f8_exponent == 0) {
194 - if ((1 << mfmt) & mantissa) {
195 - f8_exponent = 1; // denormal overflow to become normal, promote exponent
196 - }
197 - } else {
198 - if ((1 << (mfmt + 1)) & mantissa) {
199 - mantissa >>= 1;
200 - f8_exponent++;
201 - }
197 + } else {
198 + if ((1 << (mfmt + 1)) & mantissa) {
199 + mantissa >>= 1;
200 + f8_exponent++;
201 }
202 + }
203
204 - mantissa >>= (mfmt - wm);
205 -
206 - // above range: quantize to maximum possible float of the same sign
207 - const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2);
208 - if (f8_exponent > max_exp) {
209 - if (clip) {
210 - mantissa = (1 << wm) - 1;
211 - f8_exponent = max_exp;
212 - } else {
213 - return signed_inf;
214 - }
215 - }
204 + mantissa >>= (mfmt - wm);
205
217 - if (f8_exponent == 0 && mantissa == 0) {
218 - return negative_zero_nan ? 0 : (sign << 7);
206 + // above range: quantize to maximum possible float of the same sign
207 + const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2);
208 + if (f8_exponent > max_exp) {
209 + if (clip) {
210 + mantissa = (1 << wm) - 1;
211 + f8_exponent = max_exp;
212 + } else {
213 + return signed_inf;
214 }
220 - mantissa &= (1 << wm) - 1;
221 - return (sign << 7) | (f8_exponent << wm) | mantissa;
215 + }
216 +
217 + if (f8_exponent == 0 && mantissa == 0) {
218 + return negative_zero_nan ? 0 : (sign << 7);
219 + }
220 + mantissa &= (1 << wm) - 1;
221 + return (sign << 7) | (f8_exponent << wm) | mantissa;
222 }
223
224 template <int we, int wm, typename T = float, bool negative_zero_nan = true>
225 -inline HIP_FP8_HOST_DEVICE T from_float8(uint8_t x)
226 -{
225 +inline HIP_FP8_HOST_DEVICE T from_float8(uint8_t x) {
226 #ifdef __HIPCC__
228 - constexpr bool is_half = std::is_same<T, _Float16>::value;
227 + constexpr bool is_half = std::is_same<T, _Float16>::value;
228 #else
230 - constexpr bool is_half = false;
229 + constexpr bool is_half = false;
230 #endif
232 - constexpr bool is_float = std::is_same<T, float>::value;
233 - static_assert(is_half || is_float, "only half and float are supported");
231 + constexpr bool is_float = std::is_same<T, float>::value;
232 + static_assert(is_half || is_float, "only half and float are supported");
233
235 - constexpr int weo = is_half ? 5 : 8;
236 - constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7);
234 + constexpr int weo = is_half ? 5 : 8;
235 + constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7);
236
238 - T fInf, fNegInf, fNaN, fNeg0;
237 + T fInf, fNegInf, fNaN, fNeg0;
238
239 #ifdef __HIPCC__
241 - if (is_half) {
242 - const uint16_t ihInf = 0x7C00;
243 - const uint16_t ihNegInf = 0xFC00;
244 - const uint16_t ihNaN = 0x7C01;
245 - const uint16_t ihNeg0 = 0x8000;
246 - fInf = reinterpret_cast<const _Float16&>(ihInf);
247 - fNegInf = reinterpret_cast<const _Float16&>(ihNegInf);
248 - fNaN = reinterpret_cast<const _Float16&>(ihNaN);
249 - fNeg0 = reinterpret_cast<const _Float16&>(ihNeg0);
250 - } else
240 + if (is_half) {
241 + const uint16_t ihInf = 0x7C00;
242 + const uint16_t ihNegInf = 0xFC00;
243 + const uint16_t ihNaN = 0x7C01;
244 + const uint16_t ihNeg0 = 0x8000;
245 + fInf = reinterpret_cast<const _Float16&>(ihInf);
246 + fNegInf = reinterpret_cast<const _Float16&>(ihNegInf);
247 + fNaN = reinterpret_cast<const _Float16&>(ihNaN);
248 + fNeg0 = reinterpret_cast<const _Float16&>(ihNeg0);
249 + } else
250 #endif
252 - if (is_float) {
253 - const uint32_t ifInf = 0x7F800000;
254 - const uint32_t ifNegInf = 0xFF800000;
255 - const uint32_t ifNaN = 0x7F800001;
256 - const uint32_t ifNeg0 = 0x80000000;
257 - fInf = reinterpret_cast<const float&>(ifInf);
258 - fNegInf = reinterpret_cast<const float&>(ifNegInf);
259 - fNaN = reinterpret_cast<const float&>(ifNaN);
260 - fNeg0 = reinterpret_cast<const float&>(ifNeg0);
261 - }
262 -
263 - if (x == 0) {
264 - return 0;
265 - }
266 -
267 - uint32_t sign = x >> 7;
268 - uint32_t mantissa = x & ((1 << wm) - 1);
269 - int exponent = (x & 0x7F) >> wm;
270 - if (negative_zero_nan) {
271 - if (x == 0x80) {
272 - return fNaN;
273 - }
274 - } else {
275 - if (x == 0x80) {
276 - return fNeg0;
277 - }
278 - if (exponent == ((1 << we) - 1)) {
279 - return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN;
280 - }
281 - }
282 - typename std::conditional<sizeof(T) == 2, uint16_t, uint32_t>::type retval;
283 - if (we == 5 && is_half && !negative_zero_nan) {
284 - retval = x << 8;
285 - return reinterpret_cast<const T&>(retval);
251 + if (is_float) {
252 + const uint32_t ifInf = 0x7F800000;
253 + const uint32_t ifNegInf = 0xFF800000;
254 + const uint32_t ifNaN = 0x7F800001;
255 + const uint32_t ifNeg0 = 0x80000000;
256 + fInf = reinterpret_cast<const float&>(ifInf);
257 + fNegInf = reinterpret_cast<const float&>(ifNegInf);
258 + fNaN = reinterpret_cast<const float&>(ifNaN);
259 + fNeg0 = reinterpret_cast<const float&>(ifNeg0);
260 + }
261 +
262 + if (x == 0) {
263 + return 0;
264 + }
265 +
266 + uint32_t sign = x >> 7;
267 + uint32_t mantissa = x & ((1 << wm) - 1);
268 + int exponent = (x & 0x7F) >> wm;
269 + if (negative_zero_nan) {
270 + if (x == 0x80) {
271 + return fNaN;
272 }
287 -
288 - const int exp_low_cutoff = (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0);
289 -
290 - // subnormal input
291 - if (exponent == 0) {
292 - // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above
293 - int sh = 1 + clz(mantissa) - (32 - wm);
294 - mantissa <<= sh;
295 - exponent += 1 - sh;
296 - mantissa &= ((1 << wm) - 1);
273 + } else {
274 + if (x == 0x80) {
275 + return fNeg0;
276 }
298 - exponent += exp_low_cutoff - 1;
299 - mantissa <<= wmo - wm;
300 -
301 - // subnormal output (occurs when T=half, we=5, negative_zero_nan=true)
302 - if (exponent <= 0) {
303 - mantissa |= 1 << wmo;
304 - mantissa >>= 1 - exponent;
305 - exponent = 0;
306 - }
307 -
308 - if (sizeof(T) == 2) {
309 - retval = (sign << 15) | (exponent << 10) | mantissa;
310 - } else {
311 - retval = (sign << 31) | (exponent << 23) | mantissa;
277 + if (exponent == ((1 << we) - 1)) {
278 + return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN;
279 }
280 + }
281 + typename std::conditional<sizeof(T) == 2, uint16_t, uint32_t>::type retval;
282 + if (we == 5 && is_half && !negative_zero_nan) {
283 + retval = x << 8;
284 return reinterpret_cast<const T&>(retval);
285 + }
286 +
287 + const int exp_low_cutoff =
288 + (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0);
289 +
290 + // subnormal input
291 + if (exponent == 0) {
292 + // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above
293 + int sh = 1 + clz(mantissa) - (32 - wm);
294 + mantissa <<= sh;
295 + exponent += 1 - sh;
296 + mantissa &= ((1 << wm) - 1);
297 + }
298 + exponent += exp_low_cutoff - 1;
299 + mantissa <<= wmo - wm;
300 +
301 + // subnormal output (occurs when T=half, we=5, negative_zero_nan=true)
302 + if (exponent <= 0) {
303 + mantissa |= 1 << wmo;
304 + mantissa >>= 1 - exponent;
305 + exponent = 0;
306 + }
307 +
308 + if (sizeof(T) == 2) {
309 + retval = (sign << 15) | (exponent << 10) | mantissa;
310 + } else {
311 + retval = (sign << 31) | (exponent << 23) | mantissa;
312 + }
313 + return reinterpret_cast<const T&>(retval);
314 }
315
316 -} // namespace hip_fp8_impl
316 +} // namespace hip_fp8_impl
csrc/quantization/fp8/amd/quant_utils.cuh
+356 -355
@@ -9,566 +9,567 @@
9 #include "../../../attention/dtype_float32.cuh"
10 #include "../../../attention/dtype_bfloat16.cuh"
11
12 -namespace vllm
13 -{
12 +namespace vllm {
13 #ifdef USE_ROCM
14
15 namespace fp8 {
17 -#ifdef ENABLE_FP8
16 + #ifdef ENABLE_FP8
17
18 template <typename Tout, typename Tin>
20 -__inline__ __device__ Tout vec_conversion(const Tin& x)
21 -{
22 - return x;
19 +__inline__ __device__ Tout vec_conversion(const Tin& x) {
20 + return x;
21 }
22
23 template <typename Tout, typename Tin>
26 -__inline__ __device__ Tout scaled_vec_conversion(const Tin& x, const float scale)
27 -{
28 - return x;
24 +__inline__ __device__ Tout scaled_vec_conversion(const Tin& x,
25 + const float scale) {
26 + return x;
27 }
28
29 // fp8 -> half
30 template <>
33 -__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
34 -{
35 - hip_fp8 f8{a, hip_fp8::from_bits()};
36 - __half_raw res;
37 - res.data = static_cast<float>(f8);
38 - return res.x;
31 +__inline__ __device__ uint16_t
32 +vec_conversion<uint16_t, uint8_t>(const uint8_t& a) {
33 + hip_fp8 f8{a, hip_fp8::from_bits()};
34 + __half_raw res;
35 + res.data = static_cast<float>(f8);
36 + return res.x;
37 }
38
39 // fp8x2 -> half2
40 template <>
43 -__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
44 -{
45 -#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
46 - const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
47 - union {
48 - __half2_raw h2r;
49 - uint32_t ui32;
50 - } tmp;
51 - tmp.h2r.x.data = f2[0];
52 - tmp.h2r.y.data = f2[1];
53 - return tmp.ui32;
54 -#else
55 - union {
56 - uint16_t u16[2];
57 - uint32_t u32;
58 - } tmp;
59 -
60 - tmp.u16[0] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a));
61 - tmp.u16[1] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U));
62 - return tmp.u32;
63 -#endif
41 +__inline__ __device__ uint32_t
42 +vec_conversion<uint32_t, uint16_t>(const uint16_t& a) {
43 + #if defined(__HIP__MI300__) && \
44 + defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
45 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
46 + union {
47 + __half2_raw h2r;
48 + uint32_t ui32;
49 + } tmp;
50 + tmp.h2r.x.data = f2[0];
51 + tmp.h2r.y.data = f2[1];
52 + return tmp.ui32;
53 + #else
54 + union {
55 + uint16_t u16[2];
56 + uint32_t u32;
57 + } tmp;
58 +
59 + tmp.u16[0] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a));
60 + tmp.u16[1] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U));
61 + return tmp.u32;
62 + #endif
63 }
64
65 // fp8x4 -> half2x2
66 template <>
68 -__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
69 -{
70 - union {
71 - uint2 u32x2;
72 - uint32_t u32[2];
73 - } tmp;
74 - tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
75 - tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
76 - return tmp.u32x2;
67 +__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a) {
68 + union {
69 + uint2 u32x2;
70 + uint32_t u32[2];
71 + } tmp;
72 + tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
73 + tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
74 + return tmp.u32x2;
75 }
76
77 // fp8x8 -> half2x4
78 template <>
81 -__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
82 -{
83 - union {
84 - uint4 u64x2;
85 - uint2 u64[2];
86 - } tmp;
87 - tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
88 - tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
89 - return tmp.u64x2;
79 +__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a) {
80 + union {
81 + uint4 u64x2;
82 + uint2 u64[2];
83 + } tmp;
84 + tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
85 + tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
86 + return tmp.u64x2;
87 }
88
89 using __nv_bfloat16 = __hip_bfloat16;
90
91 // fp8 -> __nv_bfloat16
92 template <>
96 -__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
97 -{
98 - hip_fp8 f8{a, hip_fp8::from_bits()};
99 - float f{f8};
100 - return __float2bfloat16(f);
93 +__inline__ __device__ __nv_bfloat16
94 +vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a) {
95 + hip_fp8 f8{a, hip_fp8::from_bits()};
96 + float f{f8};
97 + return __float2bfloat16(f);
98 }
99
100 using __nv_bfloat162 = __hip_bfloat162;
101
102 // fp8x2 -> __nv_bfloat162
103 template <>
107 -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
108 -{
109 - __nv_bfloat162 res;
110 - res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
111 - res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
112 - return res;
104 +__inline__ __device__ __nv_bfloat162
105 +vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a) {
106 + __nv_bfloat162 res;
107 + res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
108 + res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
109 + return res;
110 }
111
112 // fp8x4 -> bf16_4_t
113 template <>
117 -__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
118 -{
119 - bf16_4_t res;
120 - res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
121 - res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
122 - return res;
114 +__inline__ __device__ bf16_4_t
115 +vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a) {
116 + bf16_4_t res;
117 + res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
118 + res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
119 + return res;
120 }
121
122 // fp8x8 -> bf16_8_t
123 template <>
127 -__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
128 -{
129 - bf16_4_t tmp1, tmp2;
130 - tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
131 - tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
132 - bf16_8_t res;
133 - res.x = tmp1.x;
134 - res.y = tmp1.y;
135 - res.z = tmp2.x;
136 - res.w = tmp2.y;
137 - return res;
124 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a) {
125 + bf16_4_t tmp1, tmp2;
126 + tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
127 + tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
128 + bf16_8_t res;
129 + res.x = tmp1.x;
130 + res.y = tmp1.y;
131 + res.z = tmp2.x;
132 + res.w = tmp2.y;
133 + return res;
134 }
135
136 // fp8 -> float
137 template <>
142 -__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
143 -{
144 - hip_fp8 fp8{a, hip_fp8::from_bits()};
145 - return static_cast<float>(fp8);
138 +__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a) {
139 + hip_fp8 fp8{a, hip_fp8::from_bits()};
140 + return static_cast<float>(fp8);
141 }
142
143 // fp8x2 -> float2
144 template <>
150 -__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
151 -{
152 -#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
153 - float2 res;
154 - const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
155 - res.x = f2[0];
156 - res.y = f2[1];
157 - return res;
158 -#else
159 - float2 res;
160 - res.x = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a));
161 - res.y = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U));
162 - return res;
163 -#endif
145 +__inline__ __device__ float2
146 +vec_conversion<float2, uint16_t>(const uint16_t& a) {
147 + #if defined(__HIP__MI300__) && \
148 + defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
149 + float2 res;
150 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
151 + res.x = f2[0];
152 + res.y = f2[1];
153 + return res;
154 + #else
155 + float2 res;
156 + res.x = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a));
157 + res.y = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U));
158 + return res;
159 + #endif
160 }
161
162 // fp8x4 -> float4
163 template <>
168 -__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
169 -{
170 - Float4_ res;
171 - res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
172 - res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
173 - return res;
164 +__inline__ __device__ Float4_
165 +vec_conversion<Float4_, uint32_t>(const uint32_t& a) {
166 + Float4_ res;
167 + res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
168 + res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
169 + return res;
170 }
171
172 // fp8x8 -> float8
173 template <>
178 -__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
179 -{
180 - Float4_ tmp1, tmp2;
181 - tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
182 - tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
183 - Float8_ res;
184 - res.x = tmp1.x;
185 - res.y = tmp1.y;
186 - res.z = tmp2.x;
187 - res.w = tmp2.y;
188 - return res;
174 +__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a) {
175 + Float4_ tmp1, tmp2;
176 + tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
177 + tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
178 + Float8_ res;
179 + res.x = tmp1.x;
180 + res.y = tmp1.y;
181 + res.z = tmp2.x;
182 + res.w = tmp2.y;
183 + return res;
184 }
185
186 // half -> fp8
187 template <>
193 -__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
194 -{
195 - __half_raw tmp;
196 - tmp.x = a;
188 +__inline__ __device__ uint8_t
189 +vec_conversion<uint8_t, uint16_t>(const uint16_t& a) {
190 + __half_raw tmp;
191 + tmp.x = a;
192
198 - hip_fp8 f8{static_cast<float>(tmp.data)};
199 - return f8.data;
193 + hip_fp8 f8{static_cast<float>(tmp.data)};
194 + return f8.data;
195 }
196
197 // bf16 -> fp8
198 template <>
204 -__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
205 -{
206 - hip_fp8 res{__bfloat162float(a)};
207 - return res.data;
199 +__inline__ __device__ uint8_t
200 +vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a) {
201 + hip_fp8 res{__bfloat162float(a)};
202 + return res.data;
203 }
204
205 // float -> fp8
206 template <>
212 -__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
213 -{
214 - hip_fp8 f8(a);
215 - return f8.data;
207 +__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a) {
208 + hip_fp8 f8(a);
209 + return f8.data;
210 }
211
212 // fp8x4 -> float4
213 template <>
220 -__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
221 -{
222 - Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
223 - float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
224 - return res;
214 +__inline__ __device__ float4
215 +vec_conversion<float4, uint32_t>(const uint32_t& a) {
216 + Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
217 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
218 + return res;
219 }
220
221 // float2 -> half2
222 template <>
229 -__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
230 -{
231 - union {
232 - half2 float16;
233 - uint32_t uint32;
234 - };
223 +__inline__ __device__ uint32_t
224 +vec_conversion<uint32_t, float2>(const float2& a) {
225 + union {
226 + half2 float16;
227 + uint32_t uint32;
228 + };
229
236 - float16 = __float22half2_rn(a);
237 - return uint32;
230 + float16 = __float22half2_rn(a);
231 + return uint32;
232 }
233
234 // Float4 -> half2x2
235 template <>
242 -__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
243 -{
244 - uint2 b;
245 - float2 val;
246 - val.x = a.x.x;
247 - val.y = a.x.y;
248 - b.x = vec_conversion<uint32_t, float2>(val);
236 +__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a) {
237 + uint2 b;
238 + float2 val;
239 + val.x = a.x.x;
240 + val.y = a.x.y;
241 + b.x = vec_conversion<uint32_t, float2>(val);
242
250 - val.x = a.y.x;
251 - val.y = a.y.y;
252 - b.y = vec_conversion<uint32_t, float2>(val);
253 - return b;
243 + val.x = a.y.x;
244 + val.y = a.y.y;
245 + b.y = vec_conversion<uint32_t, float2>(val);
246 + return b;
247 }
248
249 // Float4 -> float4
250 template <>
258 -__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
259 -{
260 - float4 b;
261 - b.x = a.x.x;
262 - b.y = a.x.y;
263 - b.z = a.y.x;
264 - b.w = a.y.y;
265 - return b;
251 +__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a) {
252 + float4 b;
253 + b.x = a.x.x;
254 + b.y = a.x.y;
255 + b.z = a.y.x;
256 + b.w = a.y.y;
257 + return b;
258 }
259
260 // Float8 -> half2x4
261 template <>
270 -__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
271 -{
272 - uint4 b;
273 - b.x = vec_conversion<uint32_t, float2>(a.x);
274 - b.y = vec_conversion<uint32_t, float2>(a.y);
275 - b.z = vec_conversion<uint32_t, float2>(a.z);
276 - b.w = vec_conversion<uint32_t, float2>(a.w);
277 - return b;
262 +__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a) {
263 + uint4 b;
264 + b.x = vec_conversion<uint32_t, float2>(a.x);
265 + b.y = vec_conversion<uint32_t, float2>(a.y);
266 + b.z = vec_conversion<uint32_t, float2>(a.z);
267 + b.w = vec_conversion<uint32_t, float2>(a.w);
268 + return b;
269 }
270
271 // float2 -> bfloat162
272 template <>
282 -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2& a)
283 -{
284 - __nv_bfloat162 b = __float22bfloat162_rn(a);
285 - return b;
273 +__inline__ __device__ __nv_bfloat162
274 +vec_conversion<__nv_bfloat162, float2>(const float2& a) {
275 + __nv_bfloat162 b = __float22bfloat162_rn(a);
276 + return b;
277 }
278
279 // Float4 -> bfloat162x2
280 template <>
290 -__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_& a)
291 -{
292 - bf16_4_t b;
293 - b.x = __float22bfloat162_rn(a.x);
294 - b.y = __float22bfloat162_rn(a.y);
295 - return b;
281 +__inline__ __device__ bf16_4_t
282 +vec_conversion<bf16_4_t, Float4_>(const Float4_& a) {
283 + bf16_4_t b;
284 + b.x = __float22bfloat162_rn(a.x);
285 + b.y = __float22bfloat162_rn(a.y);
286 + return b;
287 }
288
289 // Float8 -> bfloat162x4
290 template <>
300 -__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_& a)
301 -{
302 - bf16_8_t b;
303 - b.x = __float22bfloat162_rn(a.x);
304 - b.y = __float22bfloat162_rn(a.y);
305 - b.z = __float22bfloat162_rn(a.z);
306 - b.w = __float22bfloat162_rn(a.w);
307 - return b;
291 +__inline__ __device__ bf16_8_t
292 +vec_conversion<bf16_8_t, Float8_>(const Float8_& a) {
293 + bf16_8_t b;
294 + b.x = __float22bfloat162_rn(a.x);
295 + b.y = __float22bfloat162_rn(a.y);
296 + b.z = __float22bfloat162_rn(a.z);
297 + b.w = __float22bfloat162_rn(a.w);
298 + return b;
299 }
300
301 +/* Scaled and vectorized conversions, for data exchange between high and low
302 + precision domains
303
311 -/* Scaled and vectorized conversions, for data exchange between high and low precision domains
312 -
313 - Convention of the scale in API, e.g: FP8_data = Quantization( High_Precision_data / scale )
314 - s.t.
315 - Quantize(HP / scale) => FP8
316 - Dequant(FP8) * scale => HP
304 + Convention of the scale in API, e.g: FP8_data = Quantization(
305 + High_Precision_data / scale ) s.t. Quantize(HP / scale) => FP8 Dequant(FP8) *
306 + scale => HP
307
308 */
309
310 // fp8 -> half
311 template <>
322 -__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(const uint8_t& a, const float scale)
323 -{
324 - hip_fp8 f8{a, hip_fp8::from_bits()};
325 - __half_raw res;
326 - res.data = static_cast<float>(f8) * scale;
327 - return res.x;
312 +__inline__ __device__ uint16_t
313 +scaled_vec_conversion<uint16_t, uint8_t>(const uint8_t& a, const float scale) {
314 + hip_fp8 f8{a, hip_fp8::from_bits()};
315 + __half_raw res;
316 + res.data = static_cast<float>(f8) * scale;
317 + return res.x;
318 }
319
320 // fp8x2 -> half2
321 template <>
332 -__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(const uint16_t& a, const float scale)
333 -{
334 -#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
335 - const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
336 - union {
337 - __half2_raw h2r;
338 - uint32_t ui32;
339 - } tmp;
340 - tmp.h2r.x.data = f2[0] * scale;
341 - tmp.h2r.y.data = f2[1] * scale;
342 - return tmp.ui32;
343 -#else
344 - union {
345 - uint16_t u16[2];
346 - uint32_t u32;
347 - } tmp;
348 -
349 - tmp.u16[0] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a), scale);
350 - tmp.u16[1] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
351 - return tmp.u32;
352 -#endif
322 +__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
323 + const uint16_t& a, const float scale) {
324 + #if defined(__HIP__MI300__) && \
325 + defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
326 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
327 + union {
328 + __half2_raw h2r;
329 + uint32_t ui32;
330 + } tmp;
331 + tmp.h2r.x.data = f2[0] * scale;
332 + tmp.h2r.y.data = f2[1] * scale;
333 + return tmp.ui32;
334 + #else
335 + union {
336 + uint16_t u16[2];
337 + uint32_t u32;
338 + } tmp;
339 +
340 + tmp.u16[0] =
341 + scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a), scale);
342 + tmp.u16[1] = scaled_vec_conversion<uint16_t, uint8_t>(
343 + static_cast<uint8_t>(a >> 8U), scale);
344 + return tmp.u32;
345 + #endif
346 }
347
348 // fp8x4 -> half2x2
349 template <>
357 -__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(const uint32_t& a, const float scale)
358 -{
359 - union {
360 - uint2 u32x2;
361 - uint32_t u32[2];
362 - } tmp;
363 - tmp.u32[0] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale);
364 - tmp.u32[1] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), scale);
365 - return tmp.u32x2;
350 +__inline__ __device__ uint2
351 +scaled_vec_conversion<uint2, uint32_t>(const uint32_t& a, const float scale) {
352 + union {
353 + uint2 u32x2;
354 + uint32_t u32[2];
355 + } tmp;
356 + tmp.u32[0] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale);
357 + tmp.u32[1] =
358 + scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), scale);
359 + return tmp.u32x2;
360 }
361
362 // fp8x8 -> half2x4
363 template <>
370 -__inline__ __device__ uint4 scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale)
371 -{
372 - union {
373 - uint4 u64x2;
374 - uint2 u64[2];
375 - } tmp;
376 - tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale);
377 - tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale);
378 - return tmp.u64x2;
364 +__inline__ __device__ uint4
365 +scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale) {
366 + union {
367 + uint4 u64x2;
368 + uint2 u64[2];
369 + } tmp;
370 + tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale);
371 + tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale);
372 + return tmp.u64x2;
373 }
374
375 using __nv_bfloat16 = __hip_bfloat16;
376
377 // fp8 -> __nv_bfloat16
378 template <>
385 -__inline__ __device__ __nv_bfloat16 scaled_vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a, const float scale)
386 -{
387 - hip_fp8 f8{a, hip_fp8::from_bits()};
388 - float f{f8};
389 - return __float2bfloat16(f * scale);
379 +__inline__ __device__ __nv_bfloat16
380 +scaled_vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a,
381 + const float scale) {
382 + hip_fp8 f8{a, hip_fp8::from_bits()};
383 + float f{f8};
384 + return __float2bfloat16(f * scale);
385 }
386
387 using __nv_bfloat162 = __hip_bfloat162;
388
389 // fp8x2 -> __nv_bfloat162
390 template <>
396 -__inline__ __device__ __nv_bfloat162 scaled_vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a, const float scale)
397 -{
398 - __nv_bfloat162 res;
399 - res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale);
400 - res.y = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), scale);
401 - return res;
391 +__inline__ __device__ __nv_bfloat162
392 +scaled_vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a,
393 + const float scale) {
394 + __nv_bfloat162 res;
395 + res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale);
396 + res.y =
397 + scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), scale);
398 + return res;
399 }
400
401 // fp8x4 -> bf16_4_t
402 template <>
406 -__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a, const float scale)
407 -{
408 - bf16_4_t res;
409 - res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale);
410 - res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), scale);
411 - return res;
403 +__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
404 + const uint32_t& a, const float scale) {
405 + bf16_4_t res;
406 + res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale);
407 + res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U),
408 + scale);
409 + return res;
410 }
411
412 // fp8x8 -> bf16_8_t
413 template <>
416 -__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(const uint2& a, const float scale)
417 -{
418 - bf16_4_t tmp1, tmp2;
419 - tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale);
420 - tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale);
421 - bf16_8_t res;
422 - res.x = tmp1.x;
423 - res.y = tmp1.y;
424 - res.z = tmp2.x;
425 - res.w = tmp2.y;
426 - return res;
414 +__inline__ __device__ bf16_8_t
415 +scaled_vec_conversion<bf16_8_t, uint2>(const uint2& a, const float scale) {
416 + bf16_4_t tmp1, tmp2;
417 + tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale);
418 + tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale);
419 + bf16_8_t res;
420 + res.x = tmp1.x;
421 + res.y = tmp1.y;
422 + res.z = tmp2.x;
423 + res.w = tmp2.y;
424 + return res;
425 }
426
427 // fp8 -> float
428 template <>
431 -__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(const uint8_t& a, const float scale)
432 -{
433 - hip_fp8 fp8{a, hip_fp8::from_bits()};
434 - return static_cast<float>(fp8) * scale;
429 +__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
430 + const uint8_t& a, const float scale) {
431 + hip_fp8 fp8{a, hip_fp8::from_bits()};
432 + return static_cast<float>(fp8) * scale;
433 }
434
435 // fp8x2 -> float2
436 template <>
439 -__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(const uint16_t& a, const float scale)
440 -{
441 -#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
442 - float2 res;
443 - const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
444 - res.x = f2[0] * scale;
445 - res.y = f2[1] * scale;
446 - return res;
447 -#else
448 - float2 res;
449 - res.x = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a), scale);
450 - res.y = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
451 - return res;
452 -#endif
437 +__inline__ __device__ float2
438 +scaled_vec_conversion<float2, uint16_t>(const uint16_t& a, const float scale) {
439 + #if defined(__HIP__MI300__) && \
440 + defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
441 + float2 res;
442 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
443 + res.x = f2[0] * scale;
444 + res.y = f2[1] * scale;
445 + return res;
446 + #else
447 + float2 res;
448 + res.x = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a), scale);
449 + res.y = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U),
450 + scale);
451 + return res;
452 + #endif
453 }
454
455 // fp8x4 -> float4
456 template <>
457 -__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(const uint32_t& a, const float scale)
458 -{
459 - Float4_ res;
460 - res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale);
461 - res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale);
462 - return res;
457 +__inline__ __device__ Float4_
458 +scaled_vec_conversion<Float4_, uint32_t>(const uint32_t& a, const float scale) {
459 + Float4_ res;
460 + res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale);
461 + res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale);
462 + return res;
463 }
464
465 // fp8x8 -> float8
466 template <>
467 -__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(const uint2& a, const float scale)
468 -{
469 - Float4_ tmp1, tmp2;
470 - tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale);
471 - tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale);
472 - Float8_ res;
473 - res.x = tmp1.x;
474 - res.y = tmp1.y;
475 - res.z = tmp2.x;
476 - res.w = tmp2.y;
477 - return res;
467 +__inline__ __device__ Float8_
468 +scaled_vec_conversion<Float8_, uint2>(const uint2& a, const float scale) {
469 + Float4_ tmp1, tmp2;
470 + tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale);
471 + tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale);
472 + Float8_ res;
473 + res.x = tmp1.x;
474 + res.y = tmp1.y;
475 + res.z = tmp2.x;
476 + res.w = tmp2.y;
477 + return res;
478 }
479
480 -
480 /* Quantize(HP / scale) => FP8 */
481
482 // TODO(Hai): vectorized to add
483
484 // half -> fp8
485 template <>
487 -__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(const uint16_t& a, const float scale)
488 -{
489 - __half_raw tmp;
490 - tmp.x = a;
486 +__inline__ __device__ uint8_t
487 +scaled_vec_conversion<uint8_t, uint16_t>(const uint16_t& a, const float scale) {
488 + __half_raw tmp;
489 + tmp.x = a;
490
492 - hip_fp8 f8{static_cast<float>(tmp.data)/scale};
493 - return f8.data;
491 + hip_fp8 f8{static_cast<float>(tmp.data) / scale};
492 + return f8.data;
493 }
494
495 // bf16 -> fp8
496 template <>
498 -__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a, const float scale)
499 -{
500 - hip_fp8 res{__bfloat162float(a)/scale};
501 - return res.data;
497 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
498 + const __nv_bfloat16& a, const float scale) {
499 + hip_fp8 res{__bfloat162float(a) / scale};
500 + return res.data;
501 }
502
503 // float -> fp8
504 template <>
506 -__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(const float& a, const float scale)
507 -{
508 - hip_fp8 f8(a/scale);
509 - return f8.data;
505 +__inline__ __device__ uint8_t
506 +scaled_vec_conversion<uint8_t, float>(const float& a, const float scale) {
507 + hip_fp8 f8(a / scale);
508 + return f8.data;
509 }
510
511 // fp8x4 -> float4
512 template <>
514 -__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(const uint32_t& a, const float scale)
515 -{
516 - Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale);
517 - float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
518 - return res;
513 +__inline__ __device__ float4
514 +scaled_vec_conversion<float4, uint32_t>(const uint32_t& a, const float scale) {
515 + Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale);
516 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
517 + return res;
518 }
520 -#endif // ENABLE_FP8
519 + #endif // ENABLE_FP8
520
521 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
523 -__inline__ __device__ Tout convert(const Tin &x) {
524 -#ifdef ENABLE_FP8
522 +__inline__ __device__ Tout convert(const Tin& x) {
523 + #ifdef ENABLE_FP8
524 if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
525 return vec_conversion<Tout, Tin>(x);
526 }
528 -#endif
527 + #endif
528 assert(false);
529 }
530
531 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
533 -__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
534 -#ifdef ENABLE_FP8
532 +__inline__ __device__ Tout scaled_convert(const Tin& x, const float scale) {
533 + #ifdef ENABLE_FP8
534 if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
535 return scaled_vec_conversion<Tout, Tin>(x, scale);
536 }
538 -#endif
537 + #endif
538 assert(false);
539 }
540
542 -// The following macro is used to dispatch the conversion function based on the
543 -// data type of the key and value cache. The FN is a macro that calls a function
544 -// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
545 -#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546 - if (KV_DTYPE == "auto") { \
547 - if (SRC_DTYPE == at::ScalarType::Float) { \
548 - FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549 - } else if (SRC_DTYPE == at::ScalarType::Half) { \
550 - FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551 - } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
552 - FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553 - } else { \
554 - TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555 - } \
556 - } else { \
557 - if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
541 + // The following macro is used to dispatch the conversion function based on
542 + // the data type of the key and value cache. The FN is a macro that calls a
543 + // function with template<typename scalar_t, typename cache_t,
544 + // Fp8KVCacheDataType kv_dt>.
545 + #define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546 + if (KV_DTYPE == "auto") { \
547 if (SRC_DTYPE == at::ScalarType::Float) { \
559 - FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548 + FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549 } else if (SRC_DTYPE == at::ScalarType::Half) { \
561 - FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
550 + FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551 } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563 - FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
552 + FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553 } else { \
554 TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555 } \
556 } else { \
568 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
569 - } \
570 - }
557 + if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
558 + if (SRC_DTYPE == at::ScalarType::Float) { \
559 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
560 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
561 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
562 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
564 + } else { \
565 + TORCH_CHECK(false, \
566 + "Unsupported input type of kv cache: ", SRC_DTYPE); \
567 + } \
568 + } else { \
569 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
570 + } \
571 + }
572
572 -} // fp8
573 -#endif // USE_ROCM
574 -} // namespace vllm
573 +} // namespace fp8
574 +#endif // USE_ROCM
575 +} // namespace vllm
csrc/quantization/fp8/common.cu
+38 -49
@@ -10,17 +10,20 @@
10 namespace vllm {
11
12 __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
13 - float old;
14 - old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) :
15 - __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value)));
13 + float old;
14 + old = (value >= 0)
15 + ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
16 + : __uint_as_float(
17 + atomicMin((unsigned int*)addr, __float_as_uint(value)));
18
17 - return old;
19 + return old;
20 }
21
22 #define FP8_E4M3_MAX std::numeric_limits<c10::Float8_e4m3fn>::max()
23
22 -template<typename scalar_t>
23 -__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(const scalar_t val, const float scale) {
24 +template <typename scalar_t>
25 +__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
26 + const scalar_t val, const float scale) {
27 float x = static_cast<float>(val) / scale;
28 float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
29 return static_cast<c10::Float8_e4m3fn>(r);
@@ -32,11 +35,10 @@ __device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(const scalar
35 // So to get the right answer, *scale needs to be initialized to
36 // a value <= 0.0 and we need to wait for all thread blocks to
37 // finish before consuming *scale.
35 -template<typename scalar_t>
36 -__global__ void segmented_max_reduction(
37 - float* __restrict__ scale,
38 - const scalar_t* __restrict__ input,
39 - int64_t num_elems) {
38 +template <typename scalar_t>
39 +__global__ void segmented_max_reduction(float* __restrict__ scale,
40 + const scalar_t* __restrict__ input,
41 + int64_t num_elems) {
42 __shared__ float cache[1024];
43 int i = blockDim.x * blockIdx.x + threadIdx.x;
44
@@ -56,7 +58,7 @@ __global__ void segmented_max_reduction(
58 int ib = blockDim.x / 2;
59 while (ib != 0) {
60 if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
59 - cache[threadIdx.x] = cache[threadIdx.x + ib];
61 + cache[threadIdx.x] = cache[threadIdx.x + ib];
62 }
63 __syncthreads();
64 ib /= 2;
@@ -64,16 +66,16 @@ __global__ void segmented_max_reduction(
66 // Finally, since cache[0] contains the maximum for this thread block,
67 // atomically write the max to the target location
68 if (threadIdx.x == 0) {
67 - atomicMaxFloat(scale, cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
69 + atomicMaxFloat(scale,
70 + cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
71 }
72 }
73
71 -template<typename scalar_t>
72 -__global__ void scaled_fp8_quant_kernel(
73 - c10::Float8_e4m3fn* __restrict__ out,
74 - const scalar_t* __restrict__ input,
75 - const float* __restrict__ scale,
76 - int64_t num_elems) {
74 +template <typename scalar_t>
75 +__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
76 + const scalar_t* __restrict__ input,
77 + const float* __restrict__ scale,
78 + int64_t num_elems) {
79 int i = blockDim.x * blockIdx.x + threadIdx.x;
80 while (i < num_elems) {
81 out[i] = scaled_fp8_conversion(input[i], *scale);
@@ -81,12 +83,11 @@ __global__ void scaled_fp8_quant_kernel(
83 }
84 }
85
84 -} // namespace vllm
86 +} // namespace vllm
87
86 -void static_scaled_fp8_quant(
87 - torch::Tensor& out, // [..., d]
88 - torch::Tensor& input, // [..., d]
89 - torch::Tensor& scale) // [1]
88 +void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
89 + torch::Tensor& input, // [..., d]
90 + torch::Tensor& scale) // [1]
91 {
92 int64_t num_tokens = input.numel() / input.size(-1);
93 int64_t num_elems = input.numel();
@@ -95,21 +96,16 @@ void static_scaled_fp8_quant(
96 const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
97 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
98 VLLM_DISPATCH_FLOATING_TYPES(
98 - input.scalar_type(),
99 - "scaled_fp8_quant_kernel",
100 - [&] {
101 - vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
102 - out.data_ptr<c10::Float8_e4m3fn>(),
103 - input.data_ptr<scalar_t>(),
104 - scale.data_ptr<float>(),
105 - num_elems);
99 + input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
100 + vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
101 + out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
102 + scale.data_ptr<float>(), num_elems);
103 });
104 }
105
109 -void dynamic_scaled_fp8_quant(
110 - torch::Tensor& out, // [..., d]
111 - torch::Tensor& input, // [..., d]
112 - torch::Tensor& scale) // [1]
106 +void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
107 + torch::Tensor& input, // [..., d]
108 + torch::Tensor& scale) // [1]
109 {
110 int64_t num_tokens = input.numel() / input.size(-1);
111 int64_t num_elems = input.numel();
@@ -118,18 +114,11 @@ void dynamic_scaled_fp8_quant(
114 const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
115 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
116 VLLM_DISPATCH_FLOATING_TYPES(
121 - input.scalar_type(),
122 - "scaled_fp8_quant_kernel",
123 - [&] {
124 - vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
125 - scale.data_ptr<float>(),
126 - input.data_ptr<scalar_t>(),
127 - num_elems);
128 - vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
129 - out.data_ptr<c10::Float8_e4m3fn>(),
130 - input.data_ptr<scalar_t>(),
131 - scale.data_ptr<float>(),
132 - num_elems);
117 + input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
118 + vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
119 + scale.data_ptr<float>(), input.data_ptr<scalar_t>(), num_elems);
120 + vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
121 + out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
122 + scale.data_ptr<float>(), num_elems);
123 });
124 }
135 -
csrc/quantization/fp8/nvidia/quant_utils.cuh
+70 -68
@@ -10,9 +10,9 @@ namespace vllm {
10 #ifndef USE_ROCM
11
12 namespace fp8 {
13 -#ifdef ENABLE_FP8
13 + #ifdef ENABLE_FP8
14
15 -#if 0 // Disable the following code to reduce the binary size.
15 + #if 0 // Disable the following code to reduce the binary size.
16 template <typename Tout, typename Tin>
17 __inline__ __device__ Tout
18 vec_conversion(const Tin &x, const __nv_fp8_interpretation_t fp8_type) {
@@ -177,13 +177,13 @@ __inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(
177 template <>
178 __inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(
179 const __nv_bfloat16 &a, const __nv_fp8_interpretation_t fp8_type) {
180 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
180 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
181 assert(false);
182 -#else
182 + #else
183 __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(
184 __nv_bfloat16_raw(a), __NV_SATFINITE, fp8_type);
185 return (uint8_t)res;
186 -#endif
186 + #endif
187 }
188
189 // float -> fp8
@@ -276,7 +276,7 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
276 from_float(b, a);
277 return b;
278 }
279 -#endif
279 + #endif
280
281 /* Scaled and vectorized conversions, for data exchange between high and low
282 precision domains Convention of the scale in API, e.g: FP8_data =
@@ -286,14 +286,14 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
286
287 template <typename Tout, typename Tin>
288 __inline__ __device__ Tout scaled_vec_conversion(
289 - const Tin &x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
289 + const Tin& x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
290 return x;
291 }
292
293 // fp8 -> half
294 template <>
295 __inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
296 - const uint8_t &a, const float scale,
296 + const uint8_t& a, const float scale,
297 const __nv_fp8_interpretation_t fp8_type) {
298 __half_raw tmp = __nv_cvt_fp8_to_halfraw(a, fp8_type);
299 return float_to_half(half_to_float(tmp.x) * scale);
@@ -302,7 +302,7 @@ __inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
302 // fp8x2 -> half2
303 template <>
304 __inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
305 - const uint16_t &a, const float scale,
305 + const uint16_t& a, const float scale,
306 const __nv_fp8_interpretation_t fp8_type) {
307 union {
308 uint16_t u16[2];
@@ -317,7 +317,7 @@ __inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
317 // fp8x4 -> half2x2
318 template <>
319 __inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
320 - const uint32_t &a, const float scale,
320 + const uint32_t& a, const float scale,
321 const __nv_fp8_interpretation_t fp8_type) {
322 union {
323 uint2 u32x2;
@@ -333,7 +333,7 @@ __inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
333 // fp8x8 -> half2x4
334 template <>
335 __inline__ __device__ uint4
336 -scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
336 +scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale,
337 const __nv_fp8_interpretation_t fp8_type) {
338 union {
339 uint4 u64x2;
@@ -348,7 +348,7 @@ scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
348 template <>
349 __inline__ __device__ __nv_bfloat16
350 scaled_vec_conversion<__nv_bfloat16, uint8_t>(
351 - const uint8_t &a, const float scale,
351 + const uint8_t& a, const float scale,
352 const __nv_fp8_interpretation_t fp8_type) {
353 // Note there is no direct convert function from fp8 to bf16.
354 // fp8 -> half
@@ -362,7 +362,7 @@ scaled_vec_conversion<__nv_bfloat16, uint8_t>(
362 template <>
363 __inline__ __device__ __nv_bfloat162
364 scaled_vec_conversion<__nv_bfloat162, uint16_t>(
365 - const uint16_t &a, const float scale,
365 + const uint16_t& a, const float scale,
366 const __nv_fp8_interpretation_t fp8_type) {
367 __nv_bfloat162 res;
368 res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale,
@@ -375,7 +375,7 @@ scaled_vec_conversion<__nv_bfloat162, uint16_t>(
375 // fp8x4 -> bf16_4_t
376 template <>
377 __inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
378 - const uint32_t &a, const float scale,
378 + const uint32_t& a, const float scale,
379 const __nv_fp8_interpretation_t fp8_type) {
380 bf16_4_t res;
381 res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale,
@@ -388,7 +388,7 @@ __inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
388 // fp8x8 -> bf16_8_t
389 template <>
390 __inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
391 - const uint2 &a, const float scale,
391 + const uint2& a, const float scale,
392 const __nv_fp8_interpretation_t fp8_type) {
393 bf16_4_t tmp1, tmp2;
394 tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale, fp8_type);
@@ -404,9 +404,8 @@ __inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
404 // fp8 -> float
405 template <>
406 __inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
407 - const uint8_t &a, const float scale,
407 + const uint8_t& a, const float scale,
408 const __nv_fp8_interpretation_t fp8_type) {
409 -
409 // fp8 -> half
410 __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
411 uint16_t tmp = res.x;
@@ -418,7 +417,7 @@ __inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
417 // fp8x2 -> float2
418 template <>
419 __inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
421 - const uint16_t &a, const float scale,
420 + const uint16_t& a, const float scale,
421 const __nv_fp8_interpretation_t fp8_type) {
422 // fp8x2 -> half2
423 uint32_t tmp = scaled_vec_conversion<uint32_t, uint16_t>(a, scale, fp8_type);
@@ -429,7 +428,7 @@ __inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
428 // fp8x4 -> float4
429 template <>
430 __inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
432 - const uint32_t &a, const float scale,
431 + const uint32_t& a, const float scale,
432 const __nv_fp8_interpretation_t fp8_type) {
433 Float4_ res;
434 res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale, fp8_type);
@@ -441,7 +440,7 @@ __inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
440 // fp8x8 -> float8
441 template <>
442 __inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
444 - const uint2 &a, const float scale,
443 + const uint2& a, const float scale,
444 const __nv_fp8_interpretation_t fp8_type) {
445 Float4_ tmp1, tmp2;
446 tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale, fp8_type);
@@ -457,7 +456,7 @@ __inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
456 // half -> fp8
457 template <>
458 __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
460 - const uint16_t &a, const float scale,
459 + const uint16_t& a, const float scale,
460 const __nv_fp8_interpretation_t fp8_type) {
461 __nv_fp8_storage_t res =
462 __nv_cvt_float_to_fp8(half_to_float(a) / scale, __NV_SATFINITE, fp8_type);
@@ -467,21 +466,21 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
466 // bf16 -> fp8
467 template <>
468 __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
470 - const __nv_bfloat16 &a, const float scale,
469 + const __nv_bfloat16& a, const float scale,
470 const __nv_fp8_interpretation_t fp8_type) {
472 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
471 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
472 assert(false);
474 -#else
473 + #else
474 __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(__bfloat162float(a) / scale,
475 __NV_SATFINITE, fp8_type);
476 return (uint8_t)res;
478 -#endif
477 + #endif
478 }
479
480 // float -> fp8
481 template <>
482 __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
484 - const float &a, const float scale,
483 + const float& a, const float scale,
484 const __nv_fp8_interpretation_t fp8_type) {
485 __nv_fp8_storage_t res =
486 __nv_cvt_float_to_fp8(a / scale, __NV_SATFINITE, fp8_type);
@@ -491,78 +490,81 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
490 // fp8x4 -> float4
491 template <>
492 __inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(
494 - const uint32_t &a, const float scale,
493 + const uint32_t& a, const float scale,
494 const __nv_fp8_interpretation_t fp8_type) {
495 Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale, fp8_type);
496 float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
497 return res;
498 }
500 -#endif // ENABLE_FP8
499 + #endif // ENABLE_FP8
500
501 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
503 -__inline__ __device__ Tout convert(const Tin &x) {
504 -#if 0 // Disable the following code to reduce the binary size.
502 +__inline__ __device__ Tout convert(const Tin& x) {
503 + #if 0 // Disable the following code to reduce the binary size.
504 if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
505 return vec_conversion<Tout, Tin>(x, __NV_E4M3);
506 } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
507 return vec_conversion<Tout, Tin>(x, __NV_E5M2);
508 }
510 -#endif
509 + #endif
510 assert(false);
511 }
512
513 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
515 -__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
516 -#ifdef ENABLE_FP8
514 +__inline__ __device__ Tout scaled_convert(const Tin& x, const float scale) {
515 + #ifdef ENABLE_FP8
516 if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
517 return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E4M3);
518 } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
519 return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E5M2);
520 }
522 -#endif
521 + #endif
522 assert(false);
523 }
524
526 -// The following macro is used to dispatch the conversion function based on the
527 -// data type of the key and value cache. The FN is a macro that calls a function
528 -// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
529 -#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530 - if (KV_DTYPE == "auto") { \
531 - if (SRC_DTYPE == at::ScalarType::Float) { \
532 - FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533 - } else if (SRC_DTYPE == at::ScalarType::Half) { \
534 - FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535 - } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
536 - FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537 - } else { \
538 - TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539 - } \
540 - } else { \
541 - if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
525 + // The following macro is used to dispatch the conversion function based on
526 + // the data type of the key and value cache. The FN is a macro that calls a
527 + // function with template<typename scalar_t, typename cache_t,
528 + // Fp8KVCacheDataType kv_dt>.
529 + #define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530 + if (KV_DTYPE == "auto") { \
531 if (SRC_DTYPE == at::ScalarType::Float) { \
543 - FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
532 + FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533 } else if (SRC_DTYPE == at::ScalarType::Half) { \
545 - FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
534 + FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535 } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547 - FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
536 + FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537 } else { \
538 TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539 } \
551 - } else if (KV_DTYPE == "fp8_e5m2") { \
552 - if (SRC_DTYPE == at::ScalarType::Float) { \
553 - FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
554 - } else if (SRC_DTYPE == at::ScalarType::Half) { \
555 - FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
556 - } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
557 - FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
540 + } else { \
541 + if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
542 + if (SRC_DTYPE == at::ScalarType::Float) { \
543 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
544 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
545 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
546 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548 + } else { \
549 + TORCH_CHECK(false, \
550 + "Unsupported input type of kv cache: ", SRC_DTYPE); \
551 + } \
552 + } else if (KV_DTYPE == "fp8_e5m2") { \
553 + if (SRC_DTYPE == at::ScalarType::Float) { \
554 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
555 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
556 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
557 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
558 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
559 + } else { \
560 + TORCH_CHECK(false, \
561 + "Unsupported input type of kv cache: ", SRC_DTYPE); \
562 + } \
563 } else { \
559 - TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
564 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
565 } \
561 - } else { \
562 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
563 - } \
564 - }
566 + }
567
566 -} // namespace fp8
567 -#endif // not USE_ROCM
568 -} // namespace vllm
568 +} // namespace fp8
569 +#endif // not USE_ROCM
570 +} // namespace vllm
csrc/quantization/gptq/compat.cuh
+35 -35
@@ -9,54 +9,54 @@ namespace vllm {
9 namespace gptq {
10 // atomicAdd for half types, to support CC < 7.x
11
12 -__device__ __forceinline__ void atomicAdd_half(half* address, half val)
13 -{
14 - unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));
15 - unsigned int old = *address_as_ui;
16 - unsigned int assumed;
12 +__device__ __forceinline__ void atomicAdd_half(half* address, half val) {
13 + unsigned int* address_as_ui =
14 + (unsigned int*)((char*)address - ((size_t)address & 2));
15 + unsigned int old = *address_as_ui;
16 + unsigned int assumed;
17
18 - do
19 - {
20 - assumed = old;
21 - __half_raw hsum;
22 - hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
23 - half tmpres = __hadd(hsum, val);
24 - hsum = __half_raw(tmpres);
25 - old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;
26 - old = atomicCAS(address_as_ui, assumed, old);
27 - }
28 - while (assumed != old);
18 + do {
19 + assumed = old;
20 + __half_raw hsum;
21 + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
22 + half tmpres = __hadd(hsum, val);
23 + hsum = __half_raw(tmpres);
24 + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16)
25 + : (old & 0xffff0000) | hsum.x;
26 + old = atomicCAS(address_as_ui, assumed, old);
27 + } while (assumed != old);
28 }
29
30 // atomicAdd for half2 types
31
33 -__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)
34 -{
35 - unsigned int* address_as_ui = (unsigned int*)address;
36 - unsigned int old = *address_as_ui;
37 - unsigned int assumed;
38 - do
39 - {
40 - assumed = old;
41 - half2 old_val = *((half2*)&old);
42 - half2 new_val = __hadd2(old_val, val);
43 - old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
44 - }
45 - while (assumed != old);
32 +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) {
33 + unsigned int* address_as_ui = (unsigned int*)address;
34 + unsigned int old = *address_as_ui;
35 + unsigned int assumed;
36 + do {
37 + assumed = old;
38 + half2 old_val = *((half2*)&old);
39 + half2 new_val = __hadd2(old_val, val);
40 + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
41 + } while (assumed != old);
42 }
43
44 //
45
46 #if defined(__CUDA_ARCH__) || defined(USE_ROCM)
51 -#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
47 + #if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
48
53 -__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }
49 +__device__ __forceinline__ void atomicAdd(half* address, half val) {
50 + atomicAdd_half(address, val);
51 +}
52
55 -#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
56 -__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }
57 -#endif
53 + #if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
54 +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) {
55 + atomicAdd_half2(address, val);
56 +}
57 + #endif
58
59 -#endif
59 + #endif
60 #endif
61
62 } // namespace gptq
csrc/quantization/gptq/matrix_view.cuh
+262 -241
@@ -1,5 +1,6 @@
1 /*
2 -Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turboderp/exllama
2 +Adapted from https://github.com/turboderp/exllamav2 and
3 +https://github.com/turboderp/exllama
4 */
5
6 #ifndef _matrix_view_cuh
@@ -13,260 +14,280 @@ Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turbo
14 namespace vllm {
15 namespace gptq {
16
16 -class MatrixView_half
17 -{
18 -public:
19 - const half* data;
20 - const int height;
21 - const int width;
22 -
23 - __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)
24 - : data(data), height(height), width(width)
25 - { }
26 -
27 - __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
28 - __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
29 - __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }
30 - __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }
31 -
32 - __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const
33 - {
34 - half2* ptr = (half2*) item_ptr(row, column);
35 - half2 i01 = ptr[0];
36 - half2 i23 = ptr[1];
37 - items[0] = __low2half(i01);
38 - items[1] = __high2half(i01);
39 - items[2] = __low2half(i23);
40 - items[3] = __high2half(i23);
41 - }
42 - __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const
43 - {
44 - half2* ptr = (half2*)item_ptr(row, column);
45 - half2 i01 = ptr[0];
46 - half2 i23 = ptr[1];
47 - items[0] = __half2float(__low2half(i01));
48 - items[1] = __half2float(__high2half(i01));
49 - items[2] = __half2float(__low2half(i23));
50 - items[3] = __half2float(__high2half(i23));
51 - }
52 -
53 - __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const
54 - {
55 - half2* ptr = (half2*)item_ptr(row, column);
56 - half2 i01 = ptr[0];
57 - half2 i23 = ptr[1];
58 - items[0] = __half2half2(__low2half(i01));
59 - items[1] = __half2half2(__high2half(i01));
60 - items[2] = __half2half2(__low2half(i23));
61 - items[3] = __half2half2(__high2half(i23));
62 - }
17 +class MatrixView_half {
18 + public:
19 + const half* data;
20 + const int height;
21 + const int width;
22 +
23 + __device__ __forceinline__ MatrixView_half(const half* data, const int height,
24 + const int width)
25 + : data(data), height(height), width(width) {}
26 +
27 + __device__ __forceinline__ half item(int row, int column) const {
28 + return data[row * width + column];
29 + }
30 + __device__ __forceinline__ half2 item_half2(int row, int column) const {
31 + return ((half2*)data)[(row * width + column) / 2];
32 + }
33 + __device__ __forceinline__ half2 item_half2half2(int row, int column) const {
34 + return __half2half2(data[row * width + column]);
35 + }
36 + __device__ __forceinline__ const half* item_ptr(int row, int column) const {
37 + return &data[row * width + column];
38 + }
39 +
40 + __device__ __forceinline__ void item4(half (&items)[4], int row,
41 + int column) const {
42 + half2* ptr = (half2*)item_ptr(row, column);
43 + half2 i01 = ptr[0];
44 + half2 i23 = ptr[1];
45 + items[0] = __low2half(i01);
46 + items[1] = __high2half(i01);
47 + items[2] = __low2half(i23);
48 + items[3] = __high2half(i23);
49 + }
50 + __device__ __forceinline__ void item4_f(float (&items)[4], int row,
51 + int column) const {
52 + half2* ptr = (half2*)item_ptr(row, column);
53 + half2 i01 = ptr[0];
54 + half2 i23 = ptr[1];
55 + items[0] = __half2float(__low2half(i01));
56 + items[1] = __half2float(__high2half(i01));
57 + items[2] = __half2float(__low2half(i23));
58 + items[3] = __half2float(__high2half(i23));
59 + }
60 +
61 + __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row,
62 + int column) const {
63 + half2* ptr = (half2*)item_ptr(row, column);
64 + half2 i01 = ptr[0];
65 + half2 i23 = ptr[1];
66 + items[0] = __half2half2(__low2half(i01));
67 + items[1] = __half2half2(__high2half(i01));
68 + items[2] = __half2half2(__low2half(i23));
69 + items[3] = __half2half2(__high2half(i23));
70 + }
71 };
72
65 -class MatrixView_half_rw
66 -{
67 -public:
68 - half* data;
69 - const int height;
70 - const int width;
71 -
72 - __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)
73 - : data(data), height(height), width(width)
74 - { }
75 -
76 - __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
77 - __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
78 - __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }
79 - __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }
80 - __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }
81 -
82 - __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)
83 - {
84 - half2 v01 = __halves2half2(v0, v1);
85 - half2 v23 = __halves2half2(v2, v3);
86 - half2* ptr = (half2*) item_ptr(row, column);
87 - ptr[0] = v01;
88 - ptr[1] = v23;
89 - }
73 +class MatrixView_half_rw {
74 + public:
75 + half* data;
76 + const int height;
77 + const int width;
78 +
79 + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height,
80 + const int width)
81 + : data(data), height(height), width(width) {}
82 +
83 + __device__ __forceinline__ half item(int row, int column) const {
84 + return data[row * width + column];
85 + }
86 + __device__ __forceinline__ half2 item_half2(int row, int column) const {
87 + return ((half2*)data)[(row * width + column) / 2];
88 + }
89 + __device__ __forceinline__ half* item_ptr(int row, int column) {
90 + return &data[row * width + column];
91 + }
92 + __device__ __forceinline__ void set(int row, int column, half value) {
93 + data[row * width + column] = value;
94 + }
95 + __device__ __forceinline__ void set_half2(int row, int column, half2 value) {
96 + ((half2*)data)[(row * width + column) / 2] = value;
97 + }
98 +
99 + __device__ __forceinline__ void set4(int row, int column, half v0, half v1,
100 + half v2, half v3) {
101 + half2 v01 = __halves2half2(v0, v1);
102 + half2 v23 = __halves2half2(v2, v3);
103 + half2* ptr = (half2*)item_ptr(row, column);
104 + ptr[0] = v01;
105 + ptr[1] = v23;
106 + }
107 };
108
92 -class MatrixView_q4_row
93 -{
94 -public:
95 - const uint32_t* data;
96 - const int height;
97 - const int width;
98 -
99 - __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)
100 - : data(data), height(height), width(width)
101 - { }
102 -
103 - __device__ __forceinline__ int item(int row, int column) const
104 - {
105 - int shift = (column & 0x07) * 4;
106 - return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
107 - }
108 -
109 - __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
110 - {
111 - int shift = (column & 0x07) * 4;
112 - uint32_t d = data[row * width / 8 + column / 8] >> shift;
113 - items[0] = d & 0x0f;
114 - items[1] = (d >> 4) & 0x0f;
115 - }
116 -
117 - __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
118 - {
119 - int shift = (column & 0x07) * 4;
120 - uint32_t d = data[row * width / 8 + column / 8] >> shift;
121 - items[0] = d & 0x0f;
122 - items[1] = (d >> 4) & 0x0f;
123 - items[2] = (d >> 8) & 0x0f;
124 - items[3] = (d >> 12) & 0x0f;
125 - }
109 +class MatrixView_q4_row {
110 + public:
111 + const uint32_t* data;
112 + const int height;
113 + const int width;
114 +
115 + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data,
116 + const int height,
117 + const int width)
118 + : data(data), height(height), width(width) {}
119 +
120 + __device__ __forceinline__ int item(int row, int column) const {
121 + int shift = (column & 0x07) * 4;
122 + return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
123 + }
124 +
125 + __device__ __forceinline__ void item2(int (&items)[2], int row,
126 + int column) const {
127 + int shift = (column & 0x07) * 4;
128 + uint32_t d = data[row * width / 8 + column / 8] >> shift;
129 + items[0] = d & 0x0f;
130 + items[1] = (d >> 4) & 0x0f;
131 + }
132 +
133 + __device__ __forceinline__ void item4(int (&items)[4], int row,
134 + int column) const {
135 + int shift = (column & 0x07) * 4;
136 + uint32_t d = data[row * width / 8 + column / 8] >> shift;
137 + items[0] = d & 0x0f;
138 + items[1] = (d >> 4) & 0x0f;
139 + items[2] = (d >> 8) & 0x0f;
140 + items[3] = (d >> 12) & 0x0f;
141 + }
142 };
143
128 -class MatrixView_q4_column
129 -{
130 -public:
131 - const uint32_t* data;
132 - const int height;
133 - const int width;
134 -
135 - __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width)
136 - : data(data), height(height), width(width)
137 - { }
138 -
139 - __device__ __forceinline__ int item(int row, int column) const
140 - {
141 - int shift = (row & 0x07) * 4;
142 - return (data[row / 8 * width + column] >> shift) & 0x0f;
143 - }
144 -
145 - __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { return data[row / 8 * width + column]; }
146 - __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
144 +class MatrixView_q4_column {
145 + public:
146 + const uint32_t* data;
147 + const int height;
148 + const int width;
149 +
150 + __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data,
151 + const int height,
152 + const int width)
153 + : data(data), height(height), width(width) {}
154 +
155 + __device__ __forceinline__ int item(int row, int column) const {
156 + int shift = (row & 0x07) * 4;
157 + return (data[row / 8 * width + column] >> shift) & 0x0f;
158 + }
159 +
160 + __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) {
161 + return data[row / 8 * width + column];
162 + }
163 + __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row,
164 + int column) {
165 + return &data[row / 8 * width + column];
166 + }
167 };
168
149 -class MatrixView_q2_row
150 -{
151 -public:
152 - const uint32_t* data;
153 - const int height;
154 - const int width;
155 -
156 - __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width)
157 - : data(data), height(height), width(width)
158 - { }
159 -
160 - __device__ __forceinline__ int item(int row, int column) const
161 - {
162 - int shift = (column & 0x0f) * 2;
163 - return (data[row * width / 16 + column / 16] >> shift) & 0x03;
164 - }
165 -
166 - __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
167 - {
168 - int shift = (column & 0x0f) * 2;
169 - uint32_t d = data[row * width / 16 + column / 16] >> shift;
170 - items[0] = d & 0x03;
171 - items[1] = (d >> 2) & 0x03;
172 - }
173 -
174 - __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
175 - {
176 - int shift = (column & 0x0f) * 2;
177 - uint32_t d = data[row * width / 16 + column / 16] >> shift;
178 - items[0] = d & 0x03;
179 - items[1] = (d >> 2) & 0x03;
180 - items[2] = (d >> 4) & 0x03;
181 - items[3] = (d >> 6) & 0x03;
182 - }
169 +class MatrixView_q2_row {
170 + public:
171 + const uint32_t* data;
172 + const int height;
173 + const int width;
174 +
175 + __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data,
176 + const int height,
177 + const int width)
178 + : data(data), height(height), width(width) {}
179 +
180 + __device__ __forceinline__ int item(int row, int column) const {
181 + int shift = (column & 0x0f) * 2;
182 + return (data[row * width / 16 + column / 16] >> shift) & 0x03;
183 + }
184 +
185 + __device__ __forceinline__ void item2(int (&items)[2], int row,
186 + int column) const {
187 + int shift = (column & 0x0f) * 2;
188 + uint32_t d = data[row * width / 16 + column / 16] >> shift;
189 + items[0] = d & 0x03;
190 + items[1] = (d >> 2) & 0x03;
191 + }
192 +
193 + __device__ __forceinline__ void item4(int (&items)[4], int row,
194 + int column) const {
195 + int shift = (column & 0x0f) * 2;
196 + uint32_t d = data[row * width / 16 + column / 16] >> shift;
197 + items[0] = d & 0x03;
198 + items[1] = (d >> 2) & 0x03;
199 + items[2] = (d >> 4) & 0x03;
200 + items[3] = (d >> 6) & 0x03;
201 + }
202 };
203
185 -class MatrixView_q3_row
186 -{
187 -public:
188 - const uint32_t* data;
189 - const int height;
190 - const int width;
191 -
192 - __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width)
193 - : data(data), height(height), width(width)
194 - { }
195 -
196 - __device__ __forceinline__ int item(int row, int column) const
197 - {
198 - int z_w = column * 3 / 32;
199 - int z_mod = column & 0x1f;
200 -
201 - if (z_mod == 10) {
202 - return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
203 - } else if (z_mod == 21) {
204 - return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
205 - } else if (z_mod < 10) {
206 - return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
207 - } else if (z_mod < 21) {
208 - return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
209 - } else {
210 - return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
211 - }
204 +class MatrixView_q3_row {
205 + public:
206 + const uint32_t* data;
207 + const int height;
208 + const int width;
209 +
210 + __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data,
211 + const int height,
212 + const int width)
213 + : data(data), height(height), width(width) {}
214 +
215 + __device__ __forceinline__ int item(int row, int column) const {
216 + int z_w = column * 3 / 32;
217 + int z_mod = column & 0x1f;
218 +
219 + if (z_mod == 10) {
220 + return (data[row * width * 3 / 32 + z_w] >> 30) |
221 + ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
222 + } else if (z_mod == 21) {
223 + return (data[row * width * 3 / 32 + z_w] >> 31) |
224 + ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
225 + } else if (z_mod < 10) {
226 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
227 + } else if (z_mod < 21) {
228 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
229 + } else {
230 + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
231 }
213 -
214 - __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
215 - {
216 - int shift = (column & 0x1f);
217 - uint32_t d;
218 - if (shift <= 4) {
219 - d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
220 - } else if (shift == 8) {
221 - d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
222 - } else if (shift <= 16) {
223 - d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
224 - } else if (shift == 20) {
225 - d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
226 - } else {
227 - d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
228 - }
229 - items[0] = d & 0x07;
230 - items[1] = (d >> 3) & 0x07;
231 - items[2] = (d >> 6) & 0x07;
232 - items[3] = (d >> 9) & 0x07;
232 + }
233 +
234 + __device__ __forceinline__ void item4(int (&items)[4], int row,
235 + int column) const {
236 + int shift = (column & 0x1f);
237 + uint32_t d;
238 + if (shift <= 4) {
239 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
240 + } else if (shift == 8) {
241 + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) |
242 + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
243 + } else if (shift <= 16) {
244 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
245 + } else if (shift == 20) {
246 + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) |
247 + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
248 + } else {
249 + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
250 }
251 + items[0] = d & 0x07;
252 + items[1] = (d >> 3) & 0x07;
253 + items[2] = (d >> 6) & 0x07;
254 + items[3] = (d >> 9) & 0x07;
255 + }
256 };
257
236 -class MatrixView_q8_row
237 -{
238 -public:
239 - const uint32_t* data;
240 - const int height;
241 - const int width;
242 -
243 - __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width)
244 - : data(data), height(height), width(width)
245 - { }
246 -
247 - __device__ __forceinline__ int item(int row, int column) const
248 - {
249 - int shift = (column & 0x03) * 8;
250 - return (data[row * width / 4 + column / 4] >> shift) & 0xff;
251 - }
252 -
253 - __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
254 - {
255 - int shift = (column & 0x03) * 8;
256 - uint32_t d = data[row * width / 4 + column / 4] >> shift;
257 - items[0] = d & 0xff;
258 - items[1] = (d >> 8) & 0xff;
259 - }
260 -
261 - __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
262 - {
263 - int shift = (column & 0x03) * 2;
264 - uint32_t d = data[row * width / 4 + column / 4] >> shift;
265 - items[0] = d & 0xff;
266 - items[1] = (d >> 8) & 0xff;
267 - items[2] = (d >> 16) & 0xff;
268 - items[3] = (d >> 24) & 0xff;
269 - }
258 +class MatrixView_q8_row {
259 + public:
260 + const uint32_t* data;
261 + const int height;
262 + const int width;
263 +
264 + __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data,
265 + const int height,
266 + const int width)
267 + : data(data), height(height), width(width) {}
268 +
269 + __device__ __forceinline__ int item(int row, int column) const {
270 + int shift = (column & 0x03) * 8;
271 + return (data[row * width / 4 + column / 4] >> shift) & 0xff;
272 + }
273 +
274 + __device__ __forceinline__ void item2(int (&items)[2], int row,
275 + int column) const {
276 + int shift = (column & 0x03) * 8;
277 + uint32_t d = data[row * width / 4 + column / 4] >> shift;
278 + items[0] = d & 0xff;
279 + items[1] = (d >> 8) & 0xff;
280 + }
281 +
282 + __device__ __forceinline__ void item4(int (&items)[4], int row,
283 + int column) const {
284 + int shift = (column & 0x03) * 2;
285 + uint32_t d = data[row * width / 4 + column / 4] >> shift;
286 + items[0] = d & 0xff;
287 + items[1] = (d >> 8) & 0xff;
288 + items[2] = (d >> 16) & 0xff;
289 + items[3] = (d >> 24) & 0xff;
290 + }
291 };
292
293 } // namespace gptq
csrc/quantization/gptq/q_gemm.cu
+1611 -1830
@@ -1,5 +1,6 @@
1 /*
2 -Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopqwop200/GPTQ-for-LLaMa
2 +Adapted from https://github.com/turboderp/exllamav2 and
3 +https://github.com/qwopqwop200/GPTQ-for-LLaMa
4 */
5
6 #include <cstdint>
@@ -32,2044 +33,1824 @@ namespace gptq {
33 #define DIVIDE(x, size) (((x) + (size) - 1) / (size))
34
35 #if defined(USE_ROCM)
35 -#include <hipblas/hipblas.h>
36 -__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
37 - hipblasOperation_t transA,
38 - hipblasOperation_t transB,
39 - int m,
40 - int n,
41 - int k,
42 - const half* alpha,
43 - const half* AP,
44 - int lda,
45 - const half* BP,
46 - int ldb,
47 - const half* beta,
48 - half* CP,
49 - int ldc) {
50 - return hipblasHgemm(handle, transA, transB, m, n, k,
51 - reinterpret_cast<const hipblasHalf *>(alpha),
52 - reinterpret_cast<const hipblasHalf *>(AP), lda,
53 - reinterpret_cast<const hipblasHalf *>(BP), ldb,
54 - reinterpret_cast<const hipblasHalf *>(beta),
55 - reinterpret_cast<hipblasHalf *>(CP), ldc);
36 + #include <hipblas/hipblas.h>
37 +__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(
38 + hipblasHandle_t handle, hipblasOperation_t transA,
39 + hipblasOperation_t transB, int m, int n, int k, const half* alpha,
40 + const half* AP, int lda, const half* BP, int ldb, const half* beta,
41 + half* CP, int ldc) {
42 + return hipblasHgemm(handle, transA, transB, m, n, k,
43 + reinterpret_cast<const hipblasHalf*>(alpha),
44 + reinterpret_cast<const hipblasHalf*>(AP), lda,
45 + reinterpret_cast<const hipblasHalf*>(BP), ldb,
46 + reinterpret_cast<const hipblasHalf*>(beta),
47 + reinterpret_cast<hipblasHalf*>(CP), ldc);
48 }
57 -#define hipblasHgemm __compat_hipblasHgemm
49 + #define hipblasHgemm __compat_hipblasHgemm
50
59 -// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
60 -#define rocblas_operation_none HIPBLAS_OP_N
61 -#define rocblas_hgemm __compat_hipblasHgemm
51 + // Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
52 + #define rocblas_operation_none HIPBLAS_OP_N
53 + #define rocblas_hgemm __compat_hipblasHgemm
54 #endif
55
64 -__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)
65 -{
66 - half2 result = {};
67 - const half2* a2_ptr = (const half2*)a_ptr;
68 - #pragma unroll
69 - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
70 - return __hadd2(result, g_result);
56 +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr,
57 + const half2 g_result) {
58 + half2 result = {};
59 + const half2* a2_ptr = (const half2*)a_ptr;
60 +#pragma unroll
61 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
62 + return __hadd2(result, g_result);
63 }
64
73 -__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
74 -{
75 - half2 result = {};
76 - const half2* a2_ptr = (const half2*)a_ptr;
77 - #pragma unroll
78 - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
79 - return __half2float(__low2half(result)) + __half2float(__high2half(result));
65 +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) {
66 + half2 result = {};
67 + const half2* a2_ptr = (const half2*)a_ptr;
68 +#pragma unroll
69 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
70 + return __half2float(__low2half(result)) + __half2float(__high2half(result));
71 }
72
82 -__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)
83 -{
84 - half2 result = {};
85 - const half2* a2_ptr = (const half2*)a_ptr;
86 - #pragma unroll
87 - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
88 - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
73 +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr,
74 + const half2 g_result,
75 + const half qs_h) {
76 + half2 result = {};
77 + const half2* a2_ptr = (const half2*)a_ptr;
78 +#pragma unroll
79 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
80 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
81 }
82
91 -__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)
92 -{
93 - half2 result = {};
94 - const half2* a2_ptr = (const half2*)a_ptr;
95 - #pragma unroll
96 - for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
97 - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
83 +__forceinline__ __device__ half2 dot22_16(half2 (&dq)[8], const half* a_ptr,
84 + const half2 g_result,
85 + const half qs_h) {
86 + half2 result = {};
87 + const half2* a2_ptr = (const half2*)a_ptr;
88 +#pragma unroll
89 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
90 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
91 }
92
100 -__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)
101 -{
102 - half2 result = {};
103 - const half2* a2_ptr = (const half2*)a_ptr;
104 - #pragma unroll
105 - for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
106 - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
93 +__forceinline__ __device__ half2 dot22_32(half2 (&dq)[16], const half* a_ptr,
94 + const half2 g_result,
95 + const half qs_h) {
96 + half2 result = {};
97 + const half2* a2_ptr = (const half2*)a_ptr;
98 +#pragma unroll
99 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
100 + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
101 }
102
109 -__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)
110 -{
111 - half2 result = {};
112 - const half2* a2_ptr = (const half2*)a_ptr;
113 - #pragma unroll
114 - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
115 - float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
116 - return fma(result_f, qs_f, g_result);
103 +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr,
104 + const float g_result,
105 + const float qs_f) {
106 + half2 result = {};
107 + const half2* a2_ptr = (const half2*)a_ptr;
108 +#pragma unroll
109 + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
110 + float result_f =
111 + __half2float(__low2half(result)) + __half2float(__high2half(result));
112 + return fma(result_f, qs_f, g_result);
113 }
114
119 -__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)
120 -{
121 - half2 result = {};
122 - const half2* a2_ptr = (const half2*)a_ptr;
123 - #pragma unroll
124 - for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
125 - float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
126 - return fma(result_f, qs_f, g_result);
115 +__forceinline__ __device__ float dot22_16_f(half2 (&dq)[8], const half* a_ptr,
116 + const float g_result,
117 + const float qs_f) {
118 + half2 result = {};
119 + const half2* a2_ptr = (const half2*)a_ptr;
120 +#pragma unroll
121 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
122 + float result_f =
123 + __half2float(__low2half(result)) + __half2float(__high2half(result));
124 + return fma(result_f, qs_f, g_result);
125 }
126
129 -__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)
130 -{
131 - half2 result = {};
132 - const half2* a2_ptr = (const half2*)a_ptr;
133 - #pragma unroll
134 - for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
135 - float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
136 - return fma(result_f, qs_f, g_result);
127 +__forceinline__ __device__ float dot22_32_f(half2 (&dq)[16], const half* a_ptr,
128 + const float g_result,
129 + const float qs_f) {
130 + half2 result = {};
131 + const half2* a2_ptr = (const half2*)a_ptr;
132 +#pragma unroll
133 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
134 + float result_f =
135 + __half2float(__low2half(result)) + __half2float(__high2half(result));
136 + return fma(result_f, qs_f, g_result);
137 }
138
139 -__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h)
140 -{
141 - // Use FP32 accumulator to avoid potential overflow since unscaled weights are in the range -128..127
142 -
143 - float result = {};
144 - #pragma unroll
145 - for (int i = 0; i < 4; i++)
146 - {
147 - half2 w01 = dq[i];
148 - float w0 = __low2float(w01);
149 - float w1 = __high2float(w01);
150 - float x0 = __half2float(*a_ptr++);
151 - float x1 = __half2float(*a_ptr++);
152 - result = fma(w0, x0, result);
153 - result = fma(w1, x1, result);
154 - }
155 - float qs = __half2float(qs_h);
156 - result *= qs;
157 - half result_h = __float2half_rn(result);
158 - return __hadd(result_h, g_result);
139 +__forceinline__ __device__ half dot22_8_h(half2 (&dq)[4], const half* a_ptr,
140 + const half g_result,
141 + const half qs_h) {
142 + // Use FP32 accumulator to avoid potential overflow since unscaled weights are
143 + // in the range -128..127
144 +
145 + float result = {};
146 +#pragma unroll
147 + for (int i = 0; i < 4; i++) {
148 + half2 w01 = dq[i];
149 + float w0 = __low2float(w01);
150 + float w1 = __high2float(w01);
151 + float x0 = __half2float(*a_ptr++);
152 + float x1 = __half2float(*a_ptr++);
153 + result = fma(w0, x0, result);
154 + result = fma(w1, x1, result);
155 + }
156 + float qs = __half2float(qs_h);
157 + result *= qs;
158 + half result_h = __float2half_rn(result);
159 + return __hadd(result_h, g_result);
160 }
161
161 -__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h)
162 -{
163 - half2 result = {};
164 - const half2* a2_ptr = (const half2*)a_ptr;
165 - #pragma unroll
166 - for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
167 - half result_h = __hadd(__low2half(result), __high2half(result));
168 - return __hfma(result_h, qs_h, g_result);
162 +__forceinline__ __device__ half dot22_16_h(half2 (&dq)[8], const half* a_ptr,
163 + const half g_result,
164 + const half qs_h) {
165 + half2 result = {};
166 + const half2* a2_ptr = (const half2*)a_ptr;
167 +#pragma unroll
168 + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
169 + half result_h = __hadd(__low2half(result), __high2half(result));
170 + return __hfma(result_h, qs_h, g_result);
171 }
172
171 -__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h)
172 -{
173 - half2 result = {};
174 - const half2* a2_ptr = (const half2*)a_ptr;
175 - #pragma unroll
176 - for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
177 - half result_h = __hadd(__low2half(result), __high2half(result));
178 - return __hfma(result_h, qs_h, g_result);
173 +__forceinline__ __device__ half dot22_32_h(half2 (&dq)[16], const half* a_ptr,
174 + const half g_result,
175 + const half qs_h) {
176 + half2 result = {};
177 + const half2* a2_ptr = (const half2*)a_ptr;
178 +#pragma unroll
179 + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
180 + half result_h = __hadd(__low2half(result), __high2half(result));
181 + return __hfma(result_h, qs_h, g_result);
182 }
183
181 -
182 -typedef void (*fp_gemm_half_q_half_gptq_kernel)
183 -(
184 - const half*,
185 - const uint32_t*,
186 - const uint32_t*,
187 - const half*,
188 - half*,
189 - const int,
190 - const int,
191 - const int,
192 - const int,
193 - const int*
194 -);
195 -
184 +typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,
185 + const uint32_t*, const half*,
186 + half*, const int, const int,
187 + const int, const int,
188 + const int*);
189
190 template <bool first_block, int m_count>
198 -__global__ void gemm_half_q_half_gptq_4bit_kernel
199 -(
200 - const half* __restrict__ a,
201 - const uint32_t* __restrict__ b_q_weight,
191 +__global__ void gemm_half_q_half_gptq_4bit_kernel(
192 + const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
193 const uint32_t* __restrict__ b_gptq_qzeros,
203 - const half* __restrict__ b_gptq_scales,
204 - half* __restrict__ c,
205 - const int size_m,
206 - const int size_n,
207 - const int size_k,
208 - const int groups,
209 - const int* __restrict__ b_q_perm
210 -)
211 -{
212 - MatrixView_half a_(a, size_m, size_k);
213 - MatrixView_half_rw c_(c, size_m, size_n);
214 - MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
215 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
216 -
217 - int t = threadIdx.x;
218 -
219 - // Block
220 - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
221 - int offset_m = blockIdx.y * m_count;
222 - int offset_k = blockIdx.z * BLOCK_KN_SIZE;
223 -
224 - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
225 - int end_m = min(offset_m + m_count, size_m);
226 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
227 -
228 - int n = offset_n + t * 4;
229 -
230 - // Preload block_a
231 - __shared__ half block_a[m_count][BLOCK_KN_SIZE];
232 -
233 - if (offset_k + t < end_k)
234 - {
235 - for (int m = 0; m < m_count; ++m)
236 - {
237 - const half* a_ptr = a_.item_ptr(offset_m + m, 0);
238 - half* block_a_ptr = block_a[m];
239 -
240 - half a0;
241 - if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
242 - else a0 = a_ptr[offset_k + t];
243 - block_a_ptr[t] = a0;
244 - }
194 + const half* __restrict__ b_gptq_scales, half* __restrict__ c,
195 + const int size_m, const int size_n, const int size_k, const int groups,
196 + const int* __restrict__ b_q_perm) {
197 + MatrixView_half a_(a, size_m, size_k);
198 + MatrixView_half_rw c_(c, size_m, size_n);
199 + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
200 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
201 +
202 + int t = threadIdx.x;
203 +
204 + // Block
205 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
206 + int offset_m = blockIdx.y * m_count;
207 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
208 +
209 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
210 + int end_m = min(offset_m + m_count, size_m);
211 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
212 +
213 + int n = offset_n + t * 4;
214 +
215 + // Preload block_a
216 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
217 +
218 + if (offset_k + t < end_k) {
219 + for (int m = 0; m < m_count; ++m) {
220 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
221 + half* block_a_ptr = block_a[m];
222 +
223 + half a0;
224 + if (b_q_perm)
225 + a0 = a_ptr[b_q_perm[offset_k + t]];
226 + else
227 + a0 = a_ptr[offset_k + t];
228 + block_a_ptr[t] = a0;
229 }
230 + }
231
247 - // Zero output
248 - if (n >= size_n) return;
232 + // Zero output
233 + if (n >= size_n) return;
234
250 - if (blockIdx.z == 0)
251 - {
252 - for (int m = 0; m < m_count; m++)
253 - *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
235 + if (blockIdx.z == 0) {
236 + for (int m = 0; m < m_count; m++)
237 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
238 + }
239 +
240 + __syncthreads();
241 +
242 + // Find initial group
243 + int groupsize = size_k / groups;
244 + int group = offset_k / groupsize;
245 + int nextgroup = offset_k + groupsize;
246 +
247 + // a, b offset
248 + int qk = offset_k / (32 / 4);
249 +
250 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
251 + const half* a_ptr = &block_a[0][0];
252 + int a_stride = BLOCK_KN_SIZE;
253 +
254 + // Initial group
255 + int zeros[4];
256 + float scales[4];
257 + half2 z1z16[4][2];
258 + half2 y1y16[4][2];
259 + b_gptq_qzeros_.item4(zeros, group, n);
260 + b_gptq_scales_.item4_f(scales, group, n);
261 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
262 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
263 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
264 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
265 +
266 + // Column result
267 + float block_c[m_count][4] = {};
268 +
269 + // Dequantize and multiply
270 + int k = offset_k;
271 + while (k < end_k) {
272 + if (k == nextgroup) {
273 + group++;
274 + nextgroup += groupsize;
275 + b_gptq_qzeros_.item4(zeros, group, n);
276 + b_gptq_scales_.item4_f(scales, group, n);
277 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
278 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
279 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
280 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
281 }
282
256 - __syncthreads();
257 -
258 - // Find initial group
259 - int groupsize = size_k / groups;
260 - int group = offset_k / groupsize;
261 - int nextgroup = offset_k + groupsize;
262 -
263 - // a, b offset
264 - int qk = offset_k / (32 / 4);
265 -
266 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
267 - const half* a_ptr = &block_a[0][0];
268 - int a_stride = BLOCK_KN_SIZE;
269 -
270 - // Initial group
271 - int zeros[4];
272 - float scales[4];
273 - half2 z1z16[4][2];
274 - half2 y1y16[4][2];
275 - b_gptq_qzeros_.item4(zeros, group, n);
276 - b_gptq_scales_.item4_f(scales, group, n);
277 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
278 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
279 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
280 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
281 -
282 - // Column result
283 - float block_c[m_count][4] = {};
284 -
285 - // Dequantize and multiply
286 - int k = offset_k;
287 - while (k < end_k)
288 - {
289 - if (k == nextgroup)
290 - {
291 - group++;
292 - nextgroup += groupsize;
293 - b_gptq_qzeros_.item4(zeros, group, n);
294 - b_gptq_scales_.item4_f(scales, group, n);
295 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
296 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
297 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
298 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
299 - }
300 -
301 - #pragma unroll
302 - for (int j = 0; j < 4; j++)
303 - {
304 - const int4* b_ptr4 = (int4*) b_ptr;
305 - int4 load_int4 = *b_ptr4;
306 -
307 - half2 dq[4][4];
308 - dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
309 - dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
310 - dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
311 - dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
312 -
313 - #pragma unroll
314 - for (int m = 0; m < m_count; m++)
315 - {
316 - block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);
317 - block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);
318 - block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);
319 - block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);
320 - }
321 -
322 - b_ptr += size_n;
323 - a_ptr += 8;
324 - }
325 -
326 - k += 32;
283 +#pragma unroll
284 + for (int j = 0; j < 4; j++) {
285 + const int4* b_ptr4 = (int4*)b_ptr;
286 + int4 load_int4 = *b_ptr4;
287 +
288 + half2 dq[4][4];
289 + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n,
290 + false);
291 + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n,
292 + false);
293 + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n,
294 + false);
295 + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n,
296 + false);
297 +
298 +#pragma unroll
299 + for (int m = 0; m < m_count; m++) {
300 + block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0],
301 + block_c[m][0]);
302 + block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1],
303 + block_c[m][1]);
304 + block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2],
305 + block_c[m][2]);
306 + block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3],
307 + block_c[m][3]);
308 + }
309 +
310 + b_ptr += size_n;
311 + a_ptr += 8;
312 }
313
329 - for (int m = 0; m < m_count; m++)
330 - {
331 - half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
332 - half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
333 - half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
334 - atomicAdd(out , result01);
335 - atomicAdd(out + 1, result23);
336 - }
314 + k += 32;
315 + }
316 +
317 + for (int m = 0; m < m_count; m++) {
318 + half2* out = (half2*)c_.item_ptr(offset_m + m, n);
319 + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]),
320 + __float2half_rn(block_c[m][1]));
321 + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]),
322 + __float2half_rn(block_c[m][3]));
323 + atomicAdd(out, result01);
324 + atomicAdd(out + 1, result23);
325 + }
326 }
327
328 template <bool first_block, int m_count>
340 -__global__ void gemm_half_q_half_gptq_2bit_kernel
341 -(
342 - const half* __restrict__ a,
343 - const uint32_t* __restrict__ b_q_weight,
329 +__global__ void gemm_half_q_half_gptq_2bit_kernel(
330 + const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
331 const uint32_t* __restrict__ b_gptq_qzeros,
345 - const half* __restrict__ b_gptq_scales,
346 - half* __restrict__ c,
347 - const int size_m,
348 - const int size_n,
349 - const int size_k,
350 - const int groups,
351 - const int* __restrict__ b_q_perm
352 -)
353 -{
354 - MatrixView_half a_(a, size_m, size_k);
355 - MatrixView_half_rw c_(c, size_m, size_n);
356 - MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
357 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
358 -
359 - int t = threadIdx.x;
360 -
361 - // Block
362 - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
363 - int offset_m = blockIdx.y * m_count;
364 - int offset_k = blockIdx.z * BLOCK_KN_SIZE;
365 -
366 - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
367 - int end_m = min(offset_m + m_count, size_m);
368 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
369 -
370 - int n = offset_n + t * 4;
371 -
372 - // Preload block_a
373 - __shared__ half block_a[m_count][BLOCK_KN_SIZE];
374 -
375 - if (offset_k + t < end_k)
376 - {
377 - for (int m = 0; m < m_count; ++m)
378 - {
379 - const half* a_ptr = a_.item_ptr(offset_m + m, 0);
380 - half* block_a_ptr = block_a[m];
381 -
382 - half a0;
383 - if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
384 - else a0 = a_ptr[offset_k + t];
385 - block_a_ptr[t] = a0;
386 - }
332 + const half* __restrict__ b_gptq_scales, half* __restrict__ c,
333 + const int size_m, const int size_n, const int size_k, const int groups,
334 + const int* __restrict__ b_q_perm) {
335 + MatrixView_half a_(a, size_m, size_k);
336 + MatrixView_half_rw c_(c, size_m, size_n);
337 + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
338 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
339 +
340 + int t = threadIdx.x;
341 +
342 + // Block
343 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
344 + int offset_m = blockIdx.y * m_count;
345 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
346 +
347 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
348 + int end_m = min(offset_m + m_count, size_m);
349 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
350 +
351 + int n = offset_n + t * 4;
352 +
353 + // Preload block_a
354 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
355 +
356 + if (offset_k + t < end_k) {
357 + for (int m = 0; m < m_count; ++m) {
358 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
359 + half* block_a_ptr = block_a[m];
360 +
361 + half a0;
362 + if (b_q_perm)
363 + a0 = a_ptr[b_q_perm[offset_k + t]];
364 + else
365 + a0 = a_ptr[offset_k + t];
366 + block_a_ptr[t] = a0;
367 }
368 + }
369
389 - // Zero output
390 - if (n >= size_n) return;
370 + // Zero output
371 + if (n >= size_n) return;
372
392 - if (blockIdx.z == 0)
393 - {
394 - for (int m = 0; m < m_count; m++)
395 - *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
373 + if (blockIdx.z == 0) {
374 + for (int m = 0; m < m_count; m++)
375 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
376 + }
377 +
378 + __syncthreads();
379 +
380 + // Find initial group
381 + int groupsize = size_k / groups;
382 + int group = offset_k / groupsize;
383 + int nextgroup = offset_k + groupsize;
384 +
385 + // a, b offset
386 + int qk = offset_k / (32 / 2);
387 +
388 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
389 + const half* a_ptr = &block_a[0][0];
390 + int a_stride = BLOCK_KN_SIZE;
391 +
392 + // Initial group
393 + int zeros[4];
394 + half scales[4];
395 + b_gptq_qzeros_.item4(zeros, group, n);
396 + b_gptq_scales_.item4(scales, group, n);
397 + // Column result
398 + half block_c[m_count][4] = {};
399 +
400 + // Dequantize and multiply
401 + int k = offset_k;
402 + while (k < end_k) {
403 + if (k == nextgroup) {
404 + group++;
405 + nextgroup += groupsize;
406 + b_gptq_qzeros_.item4(zeros, group, n);
407 + b_gptq_scales_.item4(scales, group, n);
408 }
409
398 - __syncthreads();
399 -
400 - // Find initial group
401 - int groupsize = size_k / groups;
402 - int group = offset_k / groupsize;
403 - int nextgroup = offset_k + groupsize;
404 -
405 - // a, b offset
406 - int qk = offset_k / (32 / 2);
407 -
408 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
409 - const half* a_ptr = &block_a[0][0];
410 - int a_stride = BLOCK_KN_SIZE;
411 -
412 - // Initial group
413 - int zeros[4];
414 - half scales[4];
415 - b_gptq_qzeros_.item4(zeros, group, n);
416 - b_gptq_scales_.item4(scales, group, n);
417 - // Column result
418 - half block_c[m_count][4] = {};
419 -
420 - // Dequantize and multiply
421 - int k = offset_k;
422 - while (k < end_k)
423 - {
424 - if (k == nextgroup)
425 - {
426 - group++;
427 - nextgroup += groupsize;
428 - b_gptq_qzeros_.item4(zeros, group, n);
429 - b_gptq_scales_.item4(scales, group, n);
430 - }
431 -
432 - #pragma unroll
433 - for (int j = 0; j < 1; j++)
434 - {
435 - const int4* b_ptr4 = (int4*) b_ptr;
436 - int4 load_int4 = *b_ptr4;
437 -
438 - half2 dq[4][8];
439 - dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
440 - dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
441 - dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
442 - dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
443 -
444 - #pragma unroll
445 - for (int m = 0; m < m_count; m++)
446 - {
447 - block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
448 - block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
449 - block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
450 - block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
451 - }
452 -
453 - b_ptr += size_n;
454 - a_ptr += 16;
455 - }
456 -
457 - k += 16;
410 +#pragma unroll
411 + for (int j = 0; j < 1; j++) {
412 + const int4* b_ptr4 = (int4*)b_ptr;
413 + int4 load_int4 = *b_ptr4;
414 +
415 + half2 dq[4][8];
416 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
417 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
418 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
419 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
420 +
421 +#pragma unroll
422 + for (int m = 0; m < m_count; m++) {
423 + block_c[m][0] =
424 + dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
425 + block_c[m][1] =
426 + dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
427 + block_c[m][2] =
428 + dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
429 + block_c[m][3] =
430 + dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
431 + }
432 +
433 + b_ptr += size_n;
434 + a_ptr += 16;
435 }
436
460 - for (int m = 0; m < m_count; m++)
461 - {
462 - half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
463 - half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
464 - half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
465 - atomicAdd(out , result01);
466 - atomicAdd(out + 1, result23);
467 - }
437 + k += 16;
438 + }
439 +
440 + for (int m = 0; m < m_count; m++) {
441 + half2* out = (half2*)c_.item_ptr(offset_m + m, n);
442 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
443 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
444 + atomicAdd(out, result01);
445 + atomicAdd(out + 1, result23);
446 + }
447 }
448
449 template <bool first_block, int m_count>
471 -__global__ void gemm_half_q_half_gptq_3bit_kernel
472 -(
473 - const half* __restrict__ a,
474 - const uint32_t* __restrict__ b_q_weight,
450 +__global__ void gemm_half_q_half_gptq_3bit_kernel(
451 + const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
452 const uint32_t* __restrict__ b_gptq_qzeros,
476 - const half* __restrict__ b_gptq_scales,
477 - half* __restrict__ c,
478 - const int size_m,
479 - const int size_n,
480 - const int size_k,
481 - const int groups,
482 - const int* __restrict__ b_q_perm
483 -)
484 -{
485 - MatrixView_half a_(a, size_m, size_k);
486 - MatrixView_half_rw c_(c, size_m, size_n);
487 - MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
488 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
489 -
490 - int t = threadIdx.x;
491 -
492 - // Block
493 - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
494 - int offset_m = blockIdx.y * m_count;
495 - int offset_k = blockIdx.z * BLOCK_KN_SIZE;
496 -
497 - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
498 - int end_m = min(offset_m + m_count, size_m);
499 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
500 -
501 - int n = offset_n + t * 4;
502 -
503 - // Preload block_a
504 - __shared__ half block_a[m_count][BLOCK_KN_SIZE];
505 -
506 - if (offset_k + t < end_k)
507 - {
508 - for (int m = 0; m < m_count; ++m)
509 - {
510 - const half* a_ptr = a_.item_ptr(offset_m + m, 0);
511 - half* block_a_ptr = block_a[m];
512 -
513 - half a0;
514 - if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
515 - else a0 = a_ptr[offset_k + t];
516 - block_a_ptr[t] = a0;
517 - }
453 + const half* __restrict__ b_gptq_scales, half* __restrict__ c,
454 + const int size_m, const int size_n, const int size_k, const int groups,
455 + const int* __restrict__ b_q_perm) {
456 + MatrixView_half a_(a, size_m, size_k);
457 + MatrixView_half_rw c_(c, size_m, size_n);
458 + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
459 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
460 +
461 + int t = threadIdx.x;
462 +
463 + // Block
464 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
465 + int offset_m = blockIdx.y * m_count;
466 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
467 +
468 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
469 + int end_m = min(offset_m + m_count, size_m);
470 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
471 +
472 + int n = offset_n + t * 4;
473 +
474 + // Preload block_a
475 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
476 +
477 + if (offset_k + t < end_k) {
478 + for (int m = 0; m < m_count; ++m) {
479 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
480 + half* block_a_ptr = block_a[m];
481 +
482 + half a0;
483 + if (b_q_perm)
484 + a0 = a_ptr[b_q_perm[offset_k + t]];
485 + else
486 + a0 = a_ptr[offset_k + t];
487 + block_a_ptr[t] = a0;
488 }
489 + }
490
520 - // Zero output
521 - if (n >= size_n) return;
491 + // Zero output
492 + if (n >= size_n) return;
493
523 - if (blockIdx.z == 0)
524 - {
525 - for (int m = 0; m < m_count; m++)
526 - *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
494 + if (blockIdx.z == 0) {
495 + for (int m = 0; m < m_count; m++)
496 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
497 + }
498 +
499 + __syncthreads();
500 +
501 + // Find initial group
502 + int groupsize = size_k / groups;
503 + int group = offset_k / groupsize;
504 + int nextgroup = offset_k + groupsize;
505 +
506 + // a, b offset
507 + int qk = offset_k / 32 * 3;
508 +
509 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
510 + const half* a_ptr = &block_a[0][0];
511 + int a_stride = BLOCK_KN_SIZE;
512 +
513 + // Initial group
514 + int zeros[4];
515 + half scales[4];
516 + b_gptq_qzeros_.item4(zeros, group, n);
517 + b_gptq_scales_.item4(scales, group, n);
518 + // Column result
519 + half block_c[m_count][4] = {};
520 +
521 + // Dequantize and multiply
522 + int k = offset_k;
523 + while (k < end_k) {
524 + if (k == nextgroup) {
525 + group++;
526 + nextgroup += groupsize;
527 + b_gptq_qzeros_.item4(zeros, group, n);
528 + b_gptq_scales_.item4(scales, group, n);
529 }
530
529 - __syncthreads();
530 -
531 - // Find initial group
532 - int groupsize = size_k / groups;
533 - int group = offset_k / groupsize;
534 - int nextgroup = offset_k + groupsize;
535 -
536 - // a, b offset
537 - int qk = offset_k / 32 * 3;
538 -
539 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
540 - const half* a_ptr = &block_a[0][0];
541 - int a_stride = BLOCK_KN_SIZE;
542 -
543 - // Initial group
544 - int zeros[4];
545 - half scales[4];
546 - b_gptq_qzeros_.item4(zeros, group, n);
547 - b_gptq_scales_.item4(scales, group, n);
548 - // Column result
549 - half block_c[m_count][4] = {};
550 -
551 - // Dequantize and multiply
552 - int k = offset_k;
553 - while (k < end_k)
554 - {
555 - if (k == nextgroup)
556 - {
557 - group++;
558 - nextgroup += groupsize;
559 - b_gptq_qzeros_.item4(zeros, group, n);
560 - b_gptq_scales_.item4(scales, group, n);
561 - }
562 -
563 - #pragma unroll
564 - for (int j = 0; j < 1; j++)
565 - {
566 - int4 load_int4[3];
567 - load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
568 - load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
569 - load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
570 -
571 - half2 dq[4][16];
572 - dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
573 - dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
574 - dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
575 - dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
576 -
577 - #pragma unroll
578 - for (int m = 0; m < m_count; m++)
579 - {
580 - block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
581 - block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
582 - block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
583 - block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
584 - }
585 - a_ptr += 32;
586 - }
587 -
588 - k += 32;
531 +#pragma unroll
532 + for (int j = 0; j < 1; j++) {
533 + int4 load_int4[3];
534 + load_int4[0] = *((int4*)b_ptr);
535 + b_ptr += size_n;
536 + load_int4[1] = *((int4*)b_ptr);
537 + b_ptr += size_n;
538 + load_int4[2] = *((int4*)b_ptr);
539 + b_ptr += size_n;
540 +
541 + half2 dq[4][16];
542 + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],
543 + size_n, zeros[0] + 1);
544 + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],
545 + size_n, zeros[1] + 1);
546 + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],
547 + size_n, zeros[2] + 1);
548 + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
549 + size_n, zeros[3] + 1);
550 +
551 +#pragma unroll
552 + for (int m = 0; m < m_count; m++) {
553 + block_c[m][0] =
554 + dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
555 + block_c[m][1] =
556 + dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
557 + block_c[m][2] =
558 + dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
559 + block_c[m][3] =
560 + dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
561 + }
562 + a_ptr += 32;
563 }
564
591 - for (int m = 0; m < m_count; m++)
592 - {
593 - half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
594 - half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
595 - half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
596 - atomicAdd(out , result01);
597 - atomicAdd(out + 1, result23);
598 - }
565 + k += 32;
566 + }
567 +
568 + for (int m = 0; m < m_count; m++) {
569 + half2* out = (half2*)c_.item_ptr(offset_m + m, n);
570 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
571 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
572 + atomicAdd(out, result01);
573 + atomicAdd(out + 1, result23);
574 + }
575 }
576
577 template <bool first_block, int m_count>
602 -__global__ void gemm_half_q_half_gptq_8bit_kernel
603 -(
604 - const half* __restrict__ a,
605 - const uint32_t* __restrict__ b_q_weight,
578 +__global__ void gemm_half_q_half_gptq_8bit_kernel(
579 + const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
580 const uint32_t* __restrict__ b_gptq_qzeros,
607 - const half* __restrict__ b_gptq_scales,
608 - half* __restrict__ c,
609 - const int size_m,
610 - const int size_n,
611 - const int size_k,
612 - const int groups,
613 - const int* __restrict__ b_q_perm
614 -)
615 -{
616 - MatrixView_half a_(a, size_m, size_k);
617 - MatrixView_half_rw c_(c, size_m, size_n);
618 - MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
619 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
620 -
621 - int t = threadIdx.x;
622 -
623 - // Block
624 - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
625 - int offset_m = blockIdx.y * m_count;
626 - int offset_k = blockIdx.z * BLOCK_KN_SIZE;
627 -
628 - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
629 - int end_m = min(offset_m + m_count, size_m);
630 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
631 -
632 - int n = offset_n + t * 4;
633 -
634 - // Preload block_a
635 - __shared__ half block_a[m_count][BLOCK_KN_SIZE];
636 -
637 - if (offset_k + t < end_k)
638 - {
639 - for (int m = 0; m < m_count; ++m)
640 - {
641 - const half* a_ptr = a_.item_ptr(offset_m + m, 0);
642 - half* block_a_ptr = block_a[m];
643 -
644 - half a0;
645 - if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
646 - else a0 = a_ptr[offset_k + t];
647 - block_a_ptr[t] = a0;
648 - }
649 - }
650 -
651 - // Zero output
652 - if (n >= size_n) return;
653 -
654 - if (blockIdx.z == 0)
655 - {
656 - for (int m = 0; m < m_count; m++)
657 - *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
581 + const half* __restrict__ b_gptq_scales, half* __restrict__ c,
582 + const int size_m, const int size_n, const int size_k, const int groups,
583 + const int* __restrict__ b_q_perm) {
584 + MatrixView_half a_(a, size_m, size_k);
585 + MatrixView_half_rw c_(c, size_m, size_n);
586 + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
587 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
588 +
589 + int t = threadIdx.x;
590 +
591 + // Block
592 + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
593 + int offset_m = blockIdx.y * m_count;
594 + int offset_k = blockIdx.z * BLOCK_KN_SIZE;
595 +
596 + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
597 + int end_m = min(offset_m + m_count, size_m);
598 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
599 +
600 + int n = offset_n + t * 4;
601 +
602 + // Preload block_a
603 + __shared__ half block_a[m_count][BLOCK_KN_SIZE];
604 +
605 + if (offset_k + t < end_k) {
606 + for (int m = 0; m < m_count; ++m) {
607 + const half* a_ptr = a_.item_ptr(offset_m + m, 0);
608 + half* block_a_ptr = block_a[m];
609 +
610 + half a0;
611 + if (b_q_perm)
612 + a0 = a_ptr[b_q_perm[offset_k + t]];
613 + else
614 + a0 = a_ptr[offset_k + t];
615 + block_a_ptr[t] = a0;
616 }
617 + }
618
660 - __syncthreads();
661 -
662 - // Find initial group
663 - int groupsize = size_k / groups;
664 - int group = offset_k / groupsize;
665 - int nextgroup = offset_k + groupsize;
666 -
667 - // a, b offset
668 - int qk = offset_k / (32 / 8);
669 -
670 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
671 - const half* a_ptr = &block_a[0][0];
672 - int a_stride = BLOCK_KN_SIZE;
673 -
674 - // Initial group
675 - int zeros[4];
676 - half scales[4];
677 - b_gptq_qzeros_.item4(zeros, group, n);
678 - b_gptq_scales_.item4(scales, group, n);
679 - // Column result
680 - half block_c[m_count][4] = {};
681 -
682 - // Dequantize and multiply
683 - int k = offset_k;
684 - while (k < end_k)
685 - {
686 - if (k == nextgroup)
687 - {
688 - group++;
689 - nextgroup += groupsize;
690 - b_gptq_qzeros_.item4(zeros, group, n);
691 - b_gptq_scales_.item4(scales, group, n);
692 - }
619 + // Zero output
620 + if (n >= size_n) return;
621
694 - #pragma unroll
695 - for (int j = 0; j < 4; j++)
696 - {
697 - int4 load_int4[2];
698 - load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
699 - load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
700 -
701 - half2 dq[4][4];
702 - dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
703 - dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
704 - dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
705 - dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
706 -
707 - for (int m = 0; m < m_count; m++)
708 - {
709 - block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
710 - block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
711 - block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
712 - block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
713 - }
714 - a_ptr += 8;
715 - }
716 - k += 32;
622 + if (blockIdx.z == 0) {
623 + for (int m = 0; m < m_count; m++)
624 + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
625 + }
626 +
627 + __syncthreads();
628 +
629 + // Find initial group
630 + int groupsize = size_k / groups;
631 + int group = offset_k / groupsize;
632 + int nextgroup = offset_k + groupsize;
633 +
634 + // a, b offset
635 + int qk = offset_k / (32 / 8);
636 +
637 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
638 + const half* a_ptr = &block_a[0][0];
639 + int a_stride = BLOCK_KN_SIZE;
640 +
641 + // Initial group
642 + int zeros[4];
643 + half scales[4];
644 + b_gptq_qzeros_.item4(zeros, group, n);
645 + b_gptq_scales_.item4(scales, group, n);
646 + // Column result
647 + half block_c[m_count][4] = {};
648 +
649 + // Dequantize and multiply
650 + int k = offset_k;
651 + while (k < end_k) {
652 + if (k == nextgroup) {
653 + group++;
654 + nextgroup += groupsize;
655 + b_gptq_qzeros_.item4(zeros, group, n);
656 + b_gptq_scales_.item4(scales, group, n);
657 }
658
719 - for (int m = 0; m < m_count; m++)
720 - {
721 - half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
722 - half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
723 - half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
724 - atomicAdd(out , result01);
725 - atomicAdd(out + 1, result23);
659 +#pragma unroll
660 + for (int j = 0; j < 4; j++) {
661 + int4 load_int4[2];
662 + load_int4[0] = *((int4*)b_ptr);
663 + b_ptr += size_n;
664 + load_int4[1] = *((int4*)b_ptr);
665 + b_ptr += size_n;
666 +
667 + half2 dq[4][4];
668 + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,
669 + zeros[0] + 1);
670 + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,
671 + zeros[1] + 1);
672 + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,
673 + zeros[2] + 1);
674 + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
675 + zeros[3] + 1);
676 +
677 + for (int m = 0; m < m_count; m++) {
678 + block_c[m][0] =
679 + dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
680 + block_c[m][1] =
681 + dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
682 + block_c[m][2] =
683 + dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
684 + block_c[m][3] =
685 + dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
686 + }
687 + a_ptr += 8;
688 }
689 + k += 32;
690 + }
691 +
692 + for (int m = 0; m < m_count; m++) {
693 + half2* out = (half2*)c_.item_ptr(offset_m + m, n);
694 + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
695 + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
696 + atomicAdd(out, result01);
697 + atomicAdd(out + 1, result23);
698 + }
699 }
700
701 fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
730 - bool first_block, const int m_count, const int bit)
731 -{
732 - #define SELECT_KERNEL(M_COUNT) \
733 - if (m_count == M_COUNT) { \
734 - if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
735 - if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
736 - if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
737 - if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
738 - }
739 - #if BLOCK_M_SIZE_MAX >= 1
740 - SELECT_KERNEL(1);
741 - #endif
742 - #if BLOCK_M_SIZE_MAX >= 2
743 - SELECT_KERNEL(2);
744 - #endif
745 - #if BLOCK_M_SIZE_MAX >= 3
746 - SELECT_KERNEL(3);
747 - #endif
748 - #if BLOCK_M_SIZE_MAX >= 4
749 - SELECT_KERNEL(4);
750 - #endif
751 - #if BLOCK_M_SIZE_MAX >= 5
752 - SELECT_KERNEL(5);
753 - #endif
754 - #if BLOCK_M_SIZE_MAX >= 6
755 - SELECT_KERNEL(6);
756 - #endif
757 - #if BLOCK_M_SIZE_MAX >= 7
758 - SELECT_KERNEL(7);
759 - #endif
760 - #if BLOCK_M_SIZE_MAX >= 8
761 - SELECT_KERNEL(8);
762 - #endif
763 - return NULL;
702 + bool first_block, const int m_count, const int bit) {
703 +#define SELECT_KERNEL(M_COUNT) \
704 + if (m_count == M_COUNT) { \
705 + if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
706 + if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
707 + if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
708 + if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
709 + }
710 +#if BLOCK_M_SIZE_MAX >= 1
711 + SELECT_KERNEL(1);
712 +#endif
713 +#if BLOCK_M_SIZE_MAX >= 2
714 + SELECT_KERNEL(2);
715 +#endif
716 +#if BLOCK_M_SIZE_MAX >= 3
717 + SELECT_KERNEL(3);
718 +#endif
719 +#if BLOCK_M_SIZE_MAX >= 4
720 + SELECT_KERNEL(4);
721 +#endif
722 +#if BLOCK_M_SIZE_MAX >= 5
723 + SELECT_KERNEL(5);
724 +#endif
725 +#if BLOCK_M_SIZE_MAX >= 6
726 + SELECT_KERNEL(6);
727 +#endif
728 +#if BLOCK_M_SIZE_MAX >= 7
729 + SELECT_KERNEL(7);
730 +#endif
731 +#if BLOCK_M_SIZE_MAX >= 8
732 + SELECT_KERNEL(8);
733 +#endif
734 + return NULL;
735 }
736
737 +void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
738 + const uint32_t* b_gptq_qzeros,
739 + const half* b_gptq_scales, const int* b_q_perm,
740 + half* c, int size_m, int size_n, int size_k,
741 + int m_count, int groups, int bit) {
742 + dim3 blockDim, gridDim;
743 + blockDim.x = BLOCK_KN_SIZE;
744 + blockDim.y = 1;
745 + blockDim.z = 1;
746 + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
747 + gridDim.y = DIVIDE(size_m, m_count);
748 + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
749 +
750 + fp_gemm_half_q_half_gptq_kernel kernel =
751 + pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
752 +
753 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
754 + kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,
755 + b_gptq_scales, c, size_m, size_n,
756 + size_k, groups, b_q_perm);
757 +}
758
767 -void gemm_half_q_half_cuda_part
768 -(
769 - const half* a,
770 - const uint32_t* b_q_weight,
771 - const uint32_t* b_gptq_qzeros,
772 - const half* b_gptq_scales,
773 - const int* b_q_perm,
774 - half* c,
775 - int size_m,
776 - int size_n,
777 - int size_k,
778 - int m_count,
779 - int groups,
780 - int bit
781 -)
782 -{
783 - dim3 blockDim, gridDim;
784 - blockDim.x = BLOCK_KN_SIZE;
785 - blockDim.y = 1;
786 - blockDim.z = 1;
787 - gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
788 - gridDim.y = DIVIDE(size_m, m_count);
789 - gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
759 +__global__ void reconstruct_exllama_8bit_kernel(
760 + const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
761 + const uint32_t* __restrict__ b_gptq_qzeros,
762 + const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
763 + const int groups, half* __restrict__ b) {
764 + MatrixView_half_rw b_(b, size_k, size_n);
765 + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
766 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
767
791 - fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
768 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
769 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
770
793 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
794 - kernel<<<gridDim, blockDim, 0, stream>>>
795 - (
796 - a,
797 - b_q_weight,
798 - b_gptq_qzeros,
799 - b_gptq_scales,
800 - c,
801 - size_m,
802 - size_n,
803 - size_k,
804 - groups,
805 - b_q_perm
806 - );
807 -}
771 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
772
773 + // Preload remapping table
774 + __shared__ int perm[BLOCK_KN_SIZE];
775 + int t = threadIdx.x;
776
810 -__global__ void reconstruct_exllama_8bit_kernel
811 -(
812 - const uint32_t* __restrict__ b_q_weight,
813 - const int* __restrict__ b_q_perm,
814 - const uint32_t* __restrict__ b_gptq_qzeros,
815 - const half* __restrict__ b_gptq_scales,
816 - const int size_k,
817 - const int size_n,
818 - const int groups,
819 - half* __restrict__ b
820 -)
821 -{
822 - MatrixView_half_rw b_(b, size_k, size_n);
823 - MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
824 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
825 -
826 - int offset_k = BLOCK_KN_SIZE * blockIdx.y;
827 - int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
828 -
829 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
830 -
831 - // Preload remapping table
832 - __shared__ int perm[BLOCK_KN_SIZE];
833 - int t = threadIdx.x;
834 -
835 - if (b_q_perm)
836 - {
837 - if (offset_k + t < size_k)
838 - perm[t] = b_q_perm[offset_k + t];
839 - }
777 + if (b_q_perm) {
778 + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
779 + }
780
841 - // Column
842 - int n = offset_n + t * 4;
843 - if (n >= size_n) return;
781 + // Column
782 + int n = offset_n + t * 4;
783 + if (n >= size_n) return;
784
845 - // Find initial group
846 - int groupsize = size_k / groups;
847 - int group = offset_k / groupsize;
848 - int nextgroup = offset_k + groupsize;
785 + // Find initial group
786 + int groupsize = size_k / groups;
787 + int group = offset_k / groupsize;
788 + int nextgroup = offset_k + groupsize;
789
850 - // b offset
851 - int qk = offset_k / (32 / 8);
790 + // b offset
791 + int qk = offset_k / (32 / 8);
792
853 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
793 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
794
855 - // Initial zeros/scale
856 - int zeros[4];
857 - half2 scales[4];
858 - b_gptq_qzeros_.item4(zeros, group, n);
859 - b_gptq_scales_.item4_h2(scales, group, n);
795 + // Initial zeros/scale
796 + int zeros[4];
797 + half2 scales[4];
798 + b_gptq_qzeros_.item4(zeros, group, n);
799 + b_gptq_scales_.item4_h2(scales, group, n);
800
861 - __syncthreads();
801 + __syncthreads();
802
863 - int k = offset_k;
864 - int lk = 0;
803 + int k = offset_k;
804 + int lk = 0;
805
866 - while (k < end_k)
867 - {
868 - if (k == nextgroup)
869 - {
870 - group++;
871 - nextgroup += groupsize;
872 - b_gptq_qzeros_.item4(zeros, group, n);
873 - b_gptq_scales_.item4_h2(scales, group, n);
874 - }
806 + while (k < end_k) {
807 + if (k == nextgroup) {
808 + group++;
809 + nextgroup += groupsize;
810 + b_gptq_qzeros_.item4(zeros, group, n);
811 + b_gptq_scales_.item4_h2(scales, group, n);
812 + }
813
876 - for (int p = 0; p < 4; p++)
877 - {
878 - int4 load_int4[2];
879 - load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
880 - load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
881 -
882 - half2 dq[4][4];
883 - dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
884 - dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
885 - dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
886 - dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
887 -
888 - //half* dqh = (half*)dq;
889 - if (b_q_perm)
890 - {
891 - for (int j = 0; j < 4; j++)
892 - {
893 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
894 - b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
895 - b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
896 - }
897 - }
898 - else
899 - {
900 - for (int j = 0; j < 4; j++)
901 - {
902 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
903 - b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
904 - b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
905 - }
906 - }
814 + for (int p = 0; p < 4; p++) {
815 + int4 load_int4[2];
816 + load_int4[0] = *((int4*)b_ptr);
817 + b_ptr += size_n;
818 + load_int4[1] = *((int4*)b_ptr);
819 + b_ptr += size_n;
820 +
821 + half2 dq[4][4];
822 + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,
823 + zeros[0] + 1);
824 + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,
825 + zeros[1] + 1);
826 + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,
827 + zeros[2] + 1);
828 + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
829 + zeros[3] + 1);
830 +
831 + // half* dqh = (half*)dq;
832 + if (b_q_perm) {
833 + for (int j = 0; j < 4; j++) {
834 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
835 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
836 + __low2half(dq[2][j]), __low2half(dq[3][j]));
837 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
838 + __high2half(dq[2][j]), __high2half(dq[3][j]));
839 }
908 - k += 32;
840 + } else {
841 + for (int j = 0; j < 4; j++) {
842 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
843 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
844 + __low2half(dq[1][j]), __low2half(dq[2][j]),
845 + __low2half(dq[3][j]));
846 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
847 + __high2half(dq[1][j]), __high2half(dq[2][j]),
848 + __high2half(dq[3][j]));
849 + }
850 + }
851 }
852 + k += 32;
853 + }
854 }
855
912 -__global__ void reconstruct_exllama_4bit_kernel
913 -(
914 - const uint32_t* __restrict__ b_q_weight,
915 - const int* __restrict__ b_q_perm,
856 +__global__ void reconstruct_exllama_4bit_kernel(
857 + const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
858 const uint32_t* __restrict__ b_gptq_qzeros,
917 - const half* __restrict__ b_gptq_scales,
918 - const int size_k,
919 - const int size_n,
920 - const int groups,
921 - half* __restrict__ b
922 -)
923 -{
924 - MatrixView_half_rw b_(b, size_k, size_n);
925 - MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
926 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
927 -
928 - int offset_k = BLOCK_KN_SIZE * blockIdx.y;
929 - int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
930 -
931 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
932 -
933 - // Preload remapping table
934 - __shared__ int perm[BLOCK_KN_SIZE];
935 - int t = threadIdx.x;
936 -
937 - if (b_q_perm)
938 - {
939 - if (offset_k + t < size_k)
940 - perm[t] = b_q_perm[offset_k + t];
859 + const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
860 + const int groups, half* __restrict__ b) {
861 + MatrixView_half_rw b_(b, size_k, size_n);
862 + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
863 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
864 +
865 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
866 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
867 +
868 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
869 +
870 + // Preload remapping table
871 + __shared__ int perm[BLOCK_KN_SIZE];
872 + int t = threadIdx.x;
873 +
874 + if (b_q_perm) {
875 + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
876 + }
877 +
878 + // Column
879 + int n = offset_n + t * 4;
880 + if (n >= size_n) return;
881 +
882 + // Find initial group
883 + int groupsize = size_k / groups;
884 + int group = offset_k / groupsize;
885 + int nextgroup = offset_k + groupsize;
886 +
887 + // b offset
888 + int qk = offset_k / (32 / 4);
889 +
890 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
891 +
892 + // Initial zeros/scale
893 + int zeros[4];
894 + half2 scales[4];
895 + half2 z1z16[4][2];
896 + half2 y1y16[4][2];
897 + b_gptq_qzeros_.item4(zeros, group, n);
898 + b_gptq_scales_.item4_h2(scales, group, n);
899 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
900 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
901 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
902 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
903 +
904 + __syncthreads();
905 +
906 + int k = offset_k;
907 + int lk = 0;
908 +
909 + while (k < end_k) {
910 + if (k == nextgroup) {
911 + group++;
912 + nextgroup += groupsize;
913 + b_gptq_qzeros_.item4(zeros, group, n);
914 + b_gptq_scales_.item4_h2(scales, group, n);
915 + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
916 + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
917 + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
918 + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
919 }
920
943 - // Column
944 - int n = offset_n + t * 4;
945 - if (n >= size_n) return;
946 -
947 - // Find initial group
948 - int groupsize = size_k / groups;
949 - int group = offset_k / groupsize;
950 - int nextgroup = offset_k + groupsize;
951 -
952 - // b offset
953 - int qk = offset_k / (32 / 4);
954 -
955 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
956 -
957 - // Initial zeros/scale
958 - int zeros[4];
959 - half2 scales[4];
960 - half2 z1z16[4][2];
961 - half2 y1y16[4][2];
962 - b_gptq_qzeros_.item4(zeros, group, n);
963 - b_gptq_scales_.item4_h2(scales, group, n);
964 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
965 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
966 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
967 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
968 -
969 - __syncthreads();
970 -
971 - int k = offset_k;
972 - int lk = 0;
973 -
974 - while (k < end_k)
975 - {
976 - if (k == nextgroup)
977 - {
978 - group++;
979 - nextgroup += groupsize;
980 - b_gptq_qzeros_.item4(zeros, group, n);
981 - b_gptq_scales_.item4_h2(scales, group, n);
982 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
983 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
984 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
985 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
921 + for (int p = 0; p < 4; p++) {
922 + half2 dq[4][4];
923 + const int4* b_ptr4 = (int4*)b_ptr;
924 + int4 load_int4 = *b_ptr4;
925 +
926 + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n,
927 + false);
928 + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n,
929 + false);
930 + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n,
931 + false);
932 + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n,
933 + false);
934 +
935 + b_ptr += size_n;
936 + // half* dqh = (half*)dq;
937 + if (b_q_perm) {
938 + for (int j = 0; j < 4; j++) {
939 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
940 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
941 + __low2half(dq[2][j]), __low2half(dq[3][j]));
942 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
943 + __high2half(dq[2][j]), __high2half(dq[3][j]));
944 }
987 -
988 - for (int p = 0; p < 4; p++)
989 - {
990 - half2 dq[4][4];
991 - const int4* b_ptr4 = (int4*) b_ptr;
992 - int4 load_int4 = *b_ptr4;
993 -
994 - dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
995 - dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
996 - dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
997 - dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
998 -
999 - b_ptr += size_n;
1000 - //half* dqh = (half*)dq;
1001 - if (b_q_perm)
1002 - {
1003 - for (int j = 0; j < 4; j++)
1004 - {
1005 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1006 - b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1007 - b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1008 - }
1009 - }
1010 - else
1011 - {
1012 - for (int j = 0; j < 4; j++)
1013 - {
1014 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1015 - b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1016 - b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1017 - }
1018 - }
945 + } else {
946 + for (int j = 0; j < 4; j++) {
947 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
948 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
949 + __low2half(dq[1][j]), __low2half(dq[2][j]),
950 + __low2half(dq[3][j]));
951 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
952 + __high2half(dq[1][j]), __high2half(dq[2][j]),
953 + __high2half(dq[3][j]));
954 }
1020 - k += 32;
955 + }
956 }
957 + k += 32;
958 + }
959 }
960
1024 -__global__ void reconstruct_exllama_3bit_kernel
1025 -(
1026 - const uint32_t* __restrict__ b_q_weight,
1027 - const int* __restrict__ b_q_perm,
961 +__global__ void reconstruct_exllama_3bit_kernel(
962 + const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
963 const uint32_t* __restrict__ b_gptq_qzeros,
1029 - const half* __restrict__ b_gptq_scales,
1030 - const int size_k,
1031 - const int size_n,
1032 - const int groups,
1033 - half* __restrict__ b
1034 -)
1035 -{
1036 - MatrixView_half_rw b_(b, size_k, size_n);
1037 - MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1038 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1039 -
1040 - int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1041 - int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1042 -
1043 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1044 -
1045 - // Preload remapping table
1046 - __shared__ int perm[BLOCK_KN_SIZE];
1047 - int t = threadIdx.x;
1048 -
1049 - if (b_q_perm)
1050 - {
1051 - if (offset_k + t < size_k)
1052 - perm[t] = b_q_perm[offset_k + t];
1053 - }
964 + const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
965 + const int groups, half* __restrict__ b) {
966 + MatrixView_half_rw b_(b, size_k, size_n);
967 + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
968 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
969
1055 - // Column
1056 - int n = offset_n + t * 4;
1057 - if (n >= size_n) return;
970 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
971 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
972
1059 - // Find initial group
1060 - int groupsize = size_k / groups;
1061 - int group = offset_k / groupsize;
1062 - int nextgroup = offset_k + groupsize;
973 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
974
1064 - // b offset
1065 - int qk = offset_k / 32* 3;
975 + // Preload remapping table
976 + __shared__ int perm[BLOCK_KN_SIZE];
977 + int t = threadIdx.x;
978
1067 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
979 + if (b_q_perm) {
980 + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
981 + }
982
1069 - // Initial zeros/scale
1070 - int zeros[4];
1071 - half2 scales[4];
1072 - b_gptq_qzeros_.item4(zeros, group, n);
1073 - b_gptq_scales_.item4_h2(scales, group, n);
983 + // Column
984 + int n = offset_n + t * 4;
985 + if (n >= size_n) return;
986
1075 - __syncthreads();
987 + // Find initial group
988 + int groupsize = size_k / groups;
989 + int group = offset_k / groupsize;
990 + int nextgroup = offset_k + groupsize;
991
1077 - int k = offset_k;
1078 - int lk = 0;
992 + // b offset
993 + int qk = offset_k / 32 * 3;
994
1080 - while (k < end_k)
1081 - {
1082 - if (k == nextgroup)
1083 - {
1084 - group++;
1085 - nextgroup += groupsize;
1086 - b_gptq_qzeros_.item4(zeros, group, n);
1087 - b_gptq_scales_.item4_h2(scales, group, n);
1088 - }
995 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
996 +
997 + // Initial zeros/scale
998 + int zeros[4];
999 + half2 scales[4];
1000 + b_gptq_qzeros_.item4(zeros, group, n);
1001 + b_gptq_scales_.item4_h2(scales, group, n);
1002 +
1003 + __syncthreads();
1004
1090 - for (int p = 0; p < 1; p++)
1091 - {
1092 - int4 load_int4[3];
1093 - load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
1094 - load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
1095 - load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
1096 -
1097 - half2 dq[4][16];
1098 - dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
1099 - dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
1100 - dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
1101 - dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
1102 -
1103 - if (b_q_perm)
1104 - {
1105 - for (int j = 0; j < 16; j++)
1106 - {
1107 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1108 - b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1109 - b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1110 - }
1111 - }
1112 - else
1113 - {
1114 - for (int j = 0; j < 16; j++)
1115 - {
1116 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1117 - b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1118 - b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1119 - }
1120 - }
1005 + int k = offset_k;
1006 + int lk = 0;
1007 +
1008 + while (k < end_k) {
1009 + if (k == nextgroup) {
1010 + group++;
1011 + nextgroup += groupsize;
1012 + b_gptq_qzeros_.item4(zeros, group, n);
1013 + b_gptq_scales_.item4_h2(scales, group, n);
1014 + }
1015 +
1016 + for (int p = 0; p < 1; p++) {
1017 + int4 load_int4[3];
1018 + load_int4[0] = *((int4*)b_ptr);
1019 + b_ptr += size_n;
1020 + load_int4[1] = *((int4*)b_ptr);
1021 + b_ptr += size_n;
1022 + load_int4[2] = *((int4*)b_ptr);
1023 + b_ptr += size_n;
1024 +
1025 + half2 dq[4][16];
1026 + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],
1027 + size_n, zeros[0] + 1);
1028 + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],
1029 + size_n, zeros[1] + 1);
1030 + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],
1031 + size_n, zeros[2] + 1);
1032 + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
1033 + size_n, zeros[3] + 1);
1034 +
1035 + if (b_q_perm) {
1036 + for (int j = 0; j < 16; j++) {
1037 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1038 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
1039 + __low2half(dq[2][j]), __low2half(dq[3][j]));
1040 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
1041 + __high2half(dq[2][j]), __high2half(dq[3][j]));
1042 + }
1043 + } else {
1044 + for (int j = 0; j < 16; j++) {
1045 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1046 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
1047 + __low2half(dq[1][j]), __low2half(dq[2][j]),
1048 + __low2half(dq[3][j]));
1049 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
1050 + __high2half(dq[1][j]), __high2half(dq[2][j]),
1051 + __high2half(dq[3][j]));
1052 }
1122 - k += 32;
1053 + }
1054 }
1055 + k += 32;
1056 + }
1057 }
1058
1126 -__global__ void reconstruct_exllama_2bit_kernel
1127 -(
1128 - const uint32_t* __restrict__ b_q_weight,
1129 - const int* __restrict__ b_q_perm,
1059 +__global__ void reconstruct_exllama_2bit_kernel(
1060 + const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
1061 const uint32_t* __restrict__ b_gptq_qzeros,
1131 - const half* __restrict__ b_gptq_scales,
1132 - const int size_k,
1133 - const int size_n,
1134 - const int groups,
1135 - half* __restrict__ b
1136 -)
1137 -{
1138 - MatrixView_half_rw b_(b, size_k, size_n);
1139 - MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1140 - MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1141 -
1142 - int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1143 - int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1144 -
1145 - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1146 -
1147 - // Preload remapping table
1148 - __shared__ int perm[BLOCK_KN_SIZE];
1149 - int t = threadIdx.x;
1150 -
1151 - if (b_q_perm)
1152 - {
1153 - if (offset_k + t < size_k)
1154 - perm[t] = b_q_perm[offset_k + t];
1155 - }
1062 + const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
1063 + const int groups, half* __restrict__ b) {
1064 + MatrixView_half_rw b_(b, size_k, size_n);
1065 + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1066 + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1067
1157 - // Column
1158 - int n = offset_n + t * 4;
1159 - if (n >= size_n) return;
1068 + int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1069 + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1070
1161 - // Find initial group
1162 - int groupsize = size_k / groups;
1163 - int group = offset_k / groupsize;
1164 - int nextgroup = offset_k + groupsize;
1071 + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1072
1166 - // b offset
1167 - int qk = offset_k / (32 / 2);
1073 + // Preload remapping table
1074 + __shared__ int perm[BLOCK_KN_SIZE];
1075 + int t = threadIdx.x;
1076
1169 - const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1077 + if (b_q_perm) {
1078 + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
1079 + }
1080
1171 - // Initial zeros/scale
1172 - int zeros[4];
1173 - half2 scales[4];
1174 - b_gptq_qzeros_.item4(zeros, group, n);
1175 - b_gptq_scales_.item4_h2(scales, group, n);
1081 + // Column
1082 + int n = offset_n + t * 4;
1083 + if (n >= size_n) return;
1084
1177 - __syncthreads();
1085 + // Find initial group
1086 + int groupsize = size_k / groups;
1087 + int group = offset_k / groupsize;
1088 + int nextgroup = offset_k + groupsize;
1089
1179 - int k = offset_k;
1180 - int lk = 0;
1090 + // b offset
1091 + int qk = offset_k / (32 / 2);
1092
1182 - while (k < end_k)
1183 - {
1184 - if (k == nextgroup)
1185 - {
1186 - group++;
1187 - nextgroup += groupsize;
1188 - b_gptq_qzeros_.item4(zeros, group, n);
1189 - b_gptq_scales_.item4_h2(scales, group, n);
1190 - }
1093 + const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1094
1192 - for (int p = 0; p < 2; p++)
1193 - {
1194 - const int4* b_ptr4 = (int4*) b_ptr;
1195 - int4 load_int4 = *b_ptr4;
1196 -
1197 - half2 dq[4][8];
1198 - dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1199 - dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1200 - dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1201 - dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1202 -
1203 - b_ptr += size_n;
1204 - //half* dqh = (half*)dq;
1205 - if (b_q_perm)
1206 - {
1207 - for (int j = 0; j < 8; j++)
1208 - {
1209 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1210 - b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1211 - b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1212 - }
1213 - }
1214 - else
1215 - {
1216 - for (int j = 0; j < 8; j++)
1217 - {
1218 - for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1219 - b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1220 - b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1221 - }
1222 - }
1223 - }
1224 - k += 32;
1225 - }
1226 -}
1095 + // Initial zeros/scale
1096 + int zeros[4];
1097 + half2 scales[4];
1098 + b_gptq_qzeros_.item4(zeros, group, n);
1099 + b_gptq_scales_.item4_h2(scales, group, n);
1100
1228 -void reconstruct_exllama
1229 -(
1230 - const uint32_t* b_q_weight,
1231 - const uint32_t* b_gptq_qzeros,
1232 - const half* b_gptq_scales,
1233 - const int* b_q_perm,
1234 - half* out,
1235 - int height,
1236 - int width,
1237 - int groups,
1238 - int bit
1239 -)
1240 -{
1241 - dim3 blockDim, gridDim;
1242 - blockDim.x = BLOCK_KN_SIZE;
1243 - blockDim.y = 1;
1244 - gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1245 - gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1101 + __syncthreads();
1102
1247 - auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1248 - if (bit == 2) {
1249 - reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1250 - } else if (bit == 3) {
1251 - reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1252 - } else if (bit == 8) {
1253 - reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1103 + int k = offset_k;
1104 + int lk = 0;
1105 +
1106 + while (k < end_k) {
1107 + if (k == nextgroup) {
1108 + group++;
1109 + nextgroup += groupsize;
1110 + b_gptq_qzeros_.item4(zeros, group, n);
1111 + b_gptq_scales_.item4_h2(scales, group, n);
1112 }
1113
1256 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1257 - reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
1258 - (
1259 - b_q_weight,
1260 - b_q_perm,
1261 - b_gptq_qzeros,
1262 - b_gptq_scales,
1263 - height,
1264 - width,
1265 - groups,
1266 - out
1267 - );
1114 + for (int p = 0; p < 2; p++) {
1115 + const int4* b_ptr4 = (int4*)b_ptr;
1116 + int4 load_int4 = *b_ptr4;
1117 +
1118 + half2 dq[4][8];
1119 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1120 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1121 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1122 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1123 +
1124 + b_ptr += size_n;
1125 + // half* dqh = (half*)dq;
1126 + if (b_q_perm) {
1127 + for (int j = 0; j < 8; j++) {
1128 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1129 + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
1130 + __low2half(dq[2][j]), __low2half(dq[3][j]));
1131 + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
1132 + __high2half(dq[2][j]), __high2half(dq[3][j]));
1133 + }
1134 + } else {
1135 + for (int j = 0; j < 8; j++) {
1136 + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1137 + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
1138 + __low2half(dq[1][j]), __low2half(dq[2][j]),
1139 + __low2half(dq[3][j]));
1140 + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
1141 + __high2half(dq[1][j]), __high2half(dq[2][j]),
1142 + __high2half(dq[3][j]));
1143 + }
1144 + }
1145 + }
1146 + k += 32;
1147 + }
1148 }
1149
1150 +void reconstruct_exllama(const uint32_t* b_q_weight,
1151 + const uint32_t* b_gptq_qzeros,
1152 + const half* b_gptq_scales, const int* b_q_perm,
1153 + half* out, int height, int width, int groups,
1154 + int bit) {
1155 + dim3 blockDim, gridDim;
1156 + blockDim.x = BLOCK_KN_SIZE;
1157 + blockDim.y = 1;
1158 + gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1159 + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1160 +
1161 + auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1162 + if (bit == 2) {
1163 + reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1164 + } else if (bit == 3) {
1165 + reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1166 + } else if (bit == 8) {
1167 + reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1168 + }
1169 +
1170 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1171 + reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(
1172 + b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,
1173 + out);
1174 +}
1175
1176 __global__ void gemm_half_q_half_alt_4bit_kernel(
1272 - const half2* __restrict__ vec,
1273 - const uint32_t* __restrict__ mat,
1274 - half* __restrict__ mul,
1275 - const half* __restrict__ scales,
1276 - const uint32_t* __restrict__ zeros,
1277 - const int* __restrict__ g_idx,
1278 - int batch,
1279 - int height,
1280 - int width
1281 -)
1282 -{
1283 - int zero_width = width / 8;
1284 - int vec_height = height * 4;
1285 - const int blockwidth2 = BLOCK_KN_SIZE / 2;
1286 - int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1287 - int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1288 - int h = BLOCK_KN_SIZE * blockIdx.z / 8;
1289 - int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
1290 - int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1291 -
1292 - __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1293 - if (threadIdx.x < h_end) {
1294 - for (int m = 0; m < b_end; ++m) {
1295 - blockvec[m][threadIdx.x] =
1296 - vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1297 - threadIdx.x];
1298 - }
1177 + const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
1178 + half* __restrict__ mul, const half* __restrict__ scales,
1179 + const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
1180 + int batch, int height, int width) {
1181 + int zero_width = width / 8;
1182 + int vec_height = height * 4;
1183 + const int blockwidth2 = BLOCK_KN_SIZE / 2;
1184 + int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1185 + int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1186 + int h = BLOCK_KN_SIZE * blockIdx.z / 8;
1187 + int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
1188 + int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1189 +
1190 + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1191 + if (threadIdx.x < h_end) {
1192 + for (int m = 0; m < b_end; ++m) {
1193 + blockvec[m][threadIdx.x] =
1194 + vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1195 + threadIdx.x];
1196 }
1300 -
1301 - __shared__ half2 deq2[256][8];
1302 - int val = threadIdx.x / 8;
1303 - int off = threadIdx.x % 8;
1304 - for (; val < 256; val += BLOCK_KN_SIZE / 8) {
1305 - deq2[val][off] = __halves2half2(
1306 - __int2half_rn(val & 0xF), __int2half_rn(val >> 4)
1307 - );
1197 + }
1198 +
1199 + __shared__ half2 deq2[256][8];
1200 + int val = threadIdx.x / 8;
1201 + int off = threadIdx.x % 8;
1202 + for (; val < 256; val += BLOCK_KN_SIZE / 8) {
1203 + deq2[val][off] =
1204 + __halves2half2(__int2half_rn(val & 0xF), __int2half_rn(val >> 4));
1205 + }
1206 +
1207 + if (blockIdx.z == 0) {
1208 + for (int m = 0; m < b_end; m++) mul[(b + m) * width + w] = __int2half_rn(0);
1209 + }
1210 + __syncthreads();
1211 +
1212 + int i = width * h + w;
1213 + int g_h = h * 8;
1214 + int k = 0;
1215 + int z_w = w / 8;
1216 + int z_mod = (w % 8) * 4;
1217 + half2 res2;
1218 + half res[BLOCK_M_SIZE_MAX] = {};
1219 +
1220 + unsigned int tmp;
1221 + while (k < h_end) {
1222 + tmp = mat[i];
1223 + half2 scales_tmp[4];
1224 + half2 zeros_tmp[4];
1225 + for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
1226 + int g = g_idx[g_h + (k + tmp_k) * 2];
1227 + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1228 + half scale_f = scales[g * width + w];
1229 + half scale_f2 = scales[g2 * width + w];
1230 + half2 scale = __halves2half2(scale_f, scale_f2);
1231 + half2 zero = __halves2half2(
1232 + __hmul(scale_f,
1233 + __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) -
1234 + 1)),
1235 + __hmul(scale_f2,
1236 + __int2half_rn(
1237 + -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1)));
1238 + scales_tmp[tmp_k] = scale;
1239 + zeros_tmp[tmp_k] = zero;
1240 }
1309 -
1310 - if (blockIdx.z == 0)
1311 - {
1312 - for (int m = 0; m < b_end; m++)
1313 - mul[(b + m) * width + w] = __int2half_rn(0);
1314 - }
1315 - __syncthreads();
1316 -
1317 - int i = width * h + w;
1318 - int g_h = h * 8;
1319 - int k = 0;
1320 - int z_w = w / 8;
1321 - int z_mod = (w % 8) * 4;
1322 - half2 res2;
1323 - half res[BLOCK_M_SIZE_MAX] = {};
1324 -
1325 - unsigned int tmp;
1326 - while (k < h_end) {
1327 - tmp = mat[i];
1328 - half2 scales_tmp[4];
1329 - half2 zeros_tmp[4];
1330 - for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
1331 - int g = g_idx[g_h + (k + tmp_k) * 2];
1332 - int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1333 - half scale_f = scales[g * width + w];
1334 - half scale_f2 = scales[g2 * width + w];
1335 - half2 scale = __halves2half2(scale_f, scale_f2);
1336 - half2 zero = __halves2half2(
1337 - __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)),
1338 - __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))
1339 - );
1340 - scales_tmp[tmp_k] = scale;
1341 - zeros_tmp[tmp_k] = zero;
1342 - }
1343 - for (int m = 0; m < b_end; m++) {
1241 + for (int m = 0; m < b_end; m++) {
1242 #ifndef USE_ROCM
1345 - res2 = {};
1243 + res2 = {};
1244 #else
1347 - res2.x = __half_as_ushort(__float2half(0));
1348 - res2.y = __half_as_ushort(__float2half(0));
1245 + res2.x = __half_as_ushort(__float2half(0));
1246 + res2.y = __half_as_ushort(__float2half(0));
1247 #endif
1350 - res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1351 - res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1352 - res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
1353 - res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
1248 + res2 = __hfma2(
1249 + __hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]),
1250 + blockvec[m][k + 0], res2);
1251 + res2 = __hfma2(
1252 + __hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]),
1253 + blockvec[m][k + 1], res2);
1254 + res2 = __hfma2(
1255 + __hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]),
1256 + blockvec[m][k + 2], res2);
1257 + res2 = __hfma2(
1258 + __hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]),
1259 + blockvec[m][k + 3], res2);
1260 #ifndef USE_ROCM
1355 - res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1261 + res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1262 #else
1357 - res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1263 + res[m] = __hadd(
1264 + res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1265 #endif
1359 - }
1360 - i += width;
1361 - k += 4;
1362 - }
1363 - for (int m = 0; m < b_end; m++) {
1364 - atomicAdd(&mul[(b + m) * width + w], res[m]);
1266 }
1267 + i += width;
1268 + k += 4;
1269 + }
1270 + for (int m = 0; m < b_end; m++) {
1271 + atomicAdd(&mul[(b + m) * width + w], res[m]);
1272 + }
1273 }
1274
1368 -
1275 __global__ void gemm_half_q_half_alt_8bit_kernel(
1370 - const half2* __restrict__ vec,
1371 - const uint32_t* __restrict__ mat,
1372 - half* __restrict__ mul,
1373 - const half* __restrict__ scales,
1374 - const uint32_t* __restrict__ zeros,
1375 - const int* __restrict__ g_idx,
1376 - int batch,
1377 - int height,
1378 - int width
1379 -)
1380 -{
1381 - int zero_width = width / 4;
1382 - int vec_height = height * 2;
1383 - const int blockwidth2 = BLOCK_KN_SIZE / 2;
1384 - int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1385 - int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1386 - int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1387 - int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1388 - int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1389 -
1390 - __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1391 - if (threadIdx.x < h_end) {
1392 - for (int m = 0; m < b_end; ++m) {
1393 - blockvec[m][threadIdx.x] =
1394 - vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1395 - threadIdx.x];
1396 - }
1276 + const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
1277 + half* __restrict__ mul, const half* __restrict__ scales,
1278 + const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
1279 + int batch, int height, int width) {
1280 + int zero_width = width / 4;
1281 + int vec_height = height * 2;
1282 + const int blockwidth2 = BLOCK_KN_SIZE / 2;
1283 + int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1284 + int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1285 + int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1286 + int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1287 + int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1288 +
1289 + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1290 + if (threadIdx.x < h_end) {
1291 + for (int m = 0; m < b_end; ++m) {
1292 + blockvec[m][threadIdx.x] =
1293 + vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1294 + threadIdx.x];
1295 }
1398 -
1399 -
1400 - if (blockIdx.z == 0)
1401 - {
1402 - for (int m = 0; m < b_end; m++)
1403 - mul[(b + m) * width + w] = __int2half_rn(0);
1296 + }
1297 +
1298 + if (blockIdx.z == 0) {
1299 + for (int m = 0; m < b_end; m++) mul[(b + m) * width + w] = __int2half_rn(0);
1300 + }
1301 + __syncthreads();
1302 +
1303 + int i = width * h + w;
1304 + int g_h = h * 4;
1305 + int k = 0;
1306 + int z_w = w / 4;
1307 + int z_mod = (w % 4) * 8;
1308 + half2 res2;
1309 + half res[BLOCK_M_SIZE_MAX] = {};
1310 +
1311 + unsigned int tmp;
1312 + while (k < h_end) {
1313 + tmp = mat[i];
1314 + half2 scales_tmp[2];
1315 + half2 zeros_tmp[2];
1316 + for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1317 + int g = g_idx[g_h + (k + tmp_k) * 2];
1318 + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1319 + half scale_f = scales[g * width + w];
1320 + half scale_f2 = scales[g2 * width + w];
1321 + half2 scale = __halves2half2(scale_f, scale_f2);
1322 + half2 zero = __halves2half2(
1323 + __hmul(scale_f,
1324 + __int2half_rn(
1325 + -((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1326 + __hmul(scale_f2,
1327 + __int2half_rn(
1328 + -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1)));
1329 + scales_tmp[tmp_k] = scale;
1330 + zeros_tmp[tmp_k] = zero;
1331 }
1405 - __syncthreads();
1406 -
1407 - int i = width * h + w;
1408 - int g_h = h * 4;
1409 - int k = 0;
1410 - int z_w = w / 4;
1411 - int z_mod = (w % 4) * 8;
1412 - half2 res2;
1413 - half res[BLOCK_M_SIZE_MAX] = {};
1414 -
1415 - unsigned int tmp;
1416 - while (k < h_end) {
1417 - tmp = mat[i];
1418 - half2 scales_tmp[2];
1419 - half2 zeros_tmp[2];
1420 - for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1421 - int g = g_idx[g_h + (k + tmp_k) * 2];
1422 - int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1423 - half scale_f = scales[g * width + w];
1424 - half scale_f2 = scales[g2 * width + w];
1425 - half2 scale = __halves2half2(scale_f, scale_f2);
1426 - half2 zero = __halves2half2(
1427 - __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1428 - __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))
1429 - );
1430 - scales_tmp[tmp_k] = scale;
1431 - zeros_tmp[tmp_k] = zero;
1432 - }
1433 - for (int m = 0; m < b_end; m++) {
1332 + for (int m = 0; m < b_end; m++) {
1333 #ifndef USE_ROCM
1435 - res2 = {};
1334 + res2 = {};
1335 #else
1437 - res2.x = __half_as_ushort(__float2half(0));
1438 - res2.y = __half_as_ushort(__float2half(0));
1336 + res2.x = __half_as_ushort(__float2half(0));
1337 + res2.y = __half_as_ushort(__float2half(0));
1338 #endif
1440 - half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF));
1441 - res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1442 - half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF));
1443 - res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1339 + half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF),
1340 + __int2half_rn((tmp >> 8) & 0xFF));
1341 + res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]),
1342 + blockvec[m][k + 0], res2);
1343 + half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF),
1344 + __int2half_rn((tmp >> 24) & 0xFF));
1345 + res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]),
1346 + blockvec[m][k + 1], res2);
1347 #ifndef USE_ROCM
1445 - res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1348 + res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1349 #else
1447 - res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1350 + res[m] = __hadd(
1351 + res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1352 #endif
1449 - }
1450 - i += width;
1451 - k += 2;
1452 - }
1453 - for (int m = 0; m < b_end; m++) {
1454 - atomicAdd(&mul[(b + m) * width + w], res[m]);
1353 }
1354 + i += width;
1355 + k += 2;
1356 + }
1357 + for (int m = 0; m < b_end; m++) {
1358 + atomicAdd(&mul[(b + m) * width + w], res[m]);
1359 + }
1360 }
1361
1458 -void gemm_half_q_half_alt
1459 -(
1460 - const half* a,
1461 - const uint32_t* b_q_weight,
1462 - const uint32_t* b_gptq_qzeros,
1463 - const half* b_gptq_scales,
1464 - const int* b_g_idx,
1465 - half* c,
1466 - int size_m,
1467 - int size_n,
1468 - int size_k,
1469 - int bit
1470 -)
1471 -{
1472 - dim3 blockDim, gridDim;
1473 - blockDim.x = BLOCK_KN_SIZE;
1474 - blockDim.y = 1;
1475 - blockDim.z = 1;
1476 - gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
1477 - gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1478 - gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1479 -
1480 - auto kernel = gemm_half_q_half_alt_4bit_kernel;
1481 - if (bit == 8) {
1482 - kernel = gemm_half_q_half_alt_8bit_kernel;
1483 - }
1484 -
1485 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1486 - kernel<<<gridDim, blockDim, 0, stream>>>
1487 - (
1488 - (const half2*) a,
1489 - b_q_weight,
1490 - c,
1491 - b_gptq_scales,
1492 - b_gptq_qzeros,
1493 - b_g_idx,
1494 - size_m,
1495 - size_k / 32 * bit,
1496 - size_n
1497 - );
1362 +void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
1363 + const uint32_t* b_gptq_qzeros,
1364 + const half* b_gptq_scales, const int* b_g_idx,
1365 + half* c, int size_m, int size_n, int size_k,
1366 + int bit) {
1367 + dim3 blockDim, gridDim;
1368 + blockDim.x = BLOCK_KN_SIZE;
1369 + blockDim.y = 1;
1370 + blockDim.z = 1;
1371 + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
1372 + gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1373 + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1374 +
1375 + auto kernel = gemm_half_q_half_alt_4bit_kernel;
1376 + if (bit == 8) {
1377 + kernel = gemm_half_q_half_alt_8bit_kernel;
1378 + }
1379 +
1380 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1381 + kernel<<<gridDim, blockDim, 0, stream>>>(
1382 + (const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,
1383 + size_m, size_k / 32 * bit, size_n);
1384 }
1385
1500 -template<class T, int bit>
1501 -__global__ void reconstruct_gptq_kernel
1502 -(
1503 - const uint32_t* __restrict__ w,
1504 - const half* __restrict__ w_scales,
1505 - const uint32_t* __restrict__ w_zeros,
1506 - const int* __restrict__ g_idx,
1507 - const int height,
1508 - const int width,
1509 - const int group,
1510 - half* __restrict__ out
1511 -)
1512 -{
1513 - // Start of block
1514 -
1515 - int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1516 - int row = blockIdx.y * 32 / bit;
1517 - if (column >= width) return;
1518 -
1519 - // Views
1520 -
1521 - MatrixView_half_rw out_(out, height, width);
1522 - MatrixView_half w_scales_(w_scales, group, width);
1523 - T w_zeros_(w_zeros, group, width);
1524 -
1525 - uint32_t w_read = w[blockIdx.y * width + column];
1526 - half* out_ptr = out_.item_ptr(row, column);
1527 -
1528 - #pragma unroll
1529 - for (int s = 0; s < 32; s += bit)
1530 - {
1531 - int group = g_idx[row + s / bit];
1532 - half w_scale = w_scales_.item(group, column);
1533 - uint32_t w_zero = w_zeros_.item(group, column) + 1;
1534 - half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale);
1535 - *out_ptr = w_item; out_ptr += out_.width;
1536 - }
1386 +template <class T, int bit>
1387 +__global__ void reconstruct_gptq_kernel(const uint32_t* __restrict__ w,
1388 + const half* __restrict__ w_scales,
1389 + const uint32_t* __restrict__ w_zeros,
1390 + const int* __restrict__ g_idx,
1391 + const int height, const int width,
1392 + const int group,
1393 + half* __restrict__ out) {
1394 + // Start of block
1395 +
1396 + int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1397 + int row = blockIdx.y * 32 / bit;
1398 + if (column >= width) return;
1399 +
1400 + // Views
1401 +
1402 + MatrixView_half_rw out_(out, height, width);
1403 + MatrixView_half w_scales_(w_scales, group, width);
1404 + T w_zeros_(w_zeros, group, width);
1405 +
1406 + uint32_t w_read = w[blockIdx.y * width + column];
1407 + half* out_ptr = out_.item_ptr(row, column);
1408 +
1409 +#pragma unroll
1410 + for (int s = 0; s < 32; s += bit) {
1411 + int group = g_idx[row + s / bit];
1412 + half w_scale = w_scales_.item(group, column);
1413 + uint32_t w_zero = w_zeros_.item(group, column) + 1;
1414 + half w_item =
1415 + __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero),
1416 + w_scale);
1417 + *out_ptr = w_item;
1418 + out_ptr += out_.width;
1419 + }
1420 }
1421
1539 -__global__ void reconstruct_gptq_3bit_kernel
1540 -(
1541 - const uint32_t* __restrict__ w,
1542 - const half* __restrict__ w_scales,
1543 - const uint32_t* __restrict__ w_zeros,
1544 - const int* __restrict__ g_idx,
1545 - const int height,
1546 - const int width,
1547 - const int group,
1548 - half* __restrict__ out
1549 -)
1550 -{
1551 - // Start of block
1552 - int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1553 - int row = blockIdx.y * 32;
1554 - if (column >= width) return;
1555 -
1556 - // Views
1557 -
1558 - MatrixView_half_rw out_(out, height, width);
1559 - MatrixView_half w_scales_(w_scales, group, width);
1560 - MatrixView_q3_row w_zeros_(w_zeros, group, width);
1561 -
1562 - uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1563 - uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1564 - uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1565 - half* out_ptr = out_.item_ptr(row, column);
1566 -
1567 - #pragma unroll
1568 - for (int i = 0; i < 32; i += 1)
1569 - {
1570 - int group = g_idx[row + i];
1571 - half w_scale = w_scales_.item(group, column);
1572 - uint32_t w_zero = w_zeros_.item(group, column) + 1;
1573 - int w_item;
1574 - if (i == 10) {
1575 - w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1576 - } else if (i == 21) {
1577 - w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1578 - } else if (i < 10) {
1579 - w_item = ((w1 >> (i * 3)) & 0x7);
1580 - } else if (i < 21) {
1581 - w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1582 - } else {
1583 - w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1584 - }
1585 - *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1586 - out_ptr += out_.width;
1422 +__global__ void reconstruct_gptq_3bit_kernel(
1423 + const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
1424 + const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
1425 + const int height, const int width, const int group,
1426 + half* __restrict__ out) {
1427 + // Start of block
1428 + int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1429 + int row = blockIdx.y * 32;
1430 + if (column >= width) return;
1431 +
1432 + // Views
1433 +
1434 + MatrixView_half_rw out_(out, height, width);
1435 + MatrixView_half w_scales_(w_scales, group, width);
1436 + MatrixView_q3_row w_zeros_(w_zeros, group, width);
1437 +
1438 + uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1439 + uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1440 + uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1441 + half* out_ptr = out_.item_ptr(row, column);
1442 +
1443 +#pragma unroll
1444 + for (int i = 0; i < 32; i += 1) {
1445 + int group = g_idx[row + i];
1446 + half w_scale = w_scales_.item(group, column);
1447 + uint32_t w_zero = w_zeros_.item(group, column) + 1;
1448 + int w_item;
1449 + if (i == 10) {
1450 + w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1451 + } else if (i == 21) {
1452 + w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1453 + } else if (i < 10) {
1454 + w_item = ((w1 >> (i * 3)) & 0x7);
1455 + } else if (i < 21) {
1456 + w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1457 + } else {
1458 + w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1459 }
1460 + *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1461 + out_ptr += out_.width;
1462 + }
1463 }
1464
1590 -void reconstruct_gptq
1591 -(
1592 - const uint32_t* b_q_weight,
1593 - const uint32_t* b_gptq_qzeros,
1594 - const half* b_gptq_scales,
1595 - const int* b_g_idx,
1596 - half* out,
1597 - int height,
1598 - int width,
1599 - int groups,
1600 - int bit
1601 -)
1602 -{
1603 - dim3 blockDim, gridDim;
1604 - blockDim.x = BLOCK_KN_SIZE;
1605 - blockDim.y = 1;
1606 - gridDim.y = DIVIDE(height, 32 / bit);
1607 - gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1608 -
1609 - auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1610 - if (bit == 2) {
1611 - kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1612 - } else if (bit == 8) {
1613 - kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1614 - } else if (bit == 3) {
1615 - kernel = reconstruct_gptq_3bit_kernel;
1616 - gridDim.y = DIVIDE(height, 32);
1617 - }
1618 -
1619 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1620 - kernel<<<gridDim, blockDim, 0, stream>>>
1621 - (
1622 - b_q_weight,
1623 - b_gptq_scales,
1624 - b_gptq_qzeros,
1625 - b_g_idx,
1626 - height,
1627 - width,
1628 - groups,
1629 - out
1630 - );
1465 +void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
1466 + const half* b_gptq_scales, const int* b_g_idx, half* out,
1467 + int height, int width, int groups, int bit) {
1468 + dim3 blockDim, gridDim;
1469 + blockDim.x = BLOCK_KN_SIZE;
1470 + blockDim.y = 1;
1471 + gridDim.y = DIVIDE(height, 32 / bit);
1472 + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1473 +
1474 + auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1475 + if (bit == 2) {
1476 + kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1477 + } else if (bit == 8) {
1478 + kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1479 + } else if (bit == 3) {
1480 + kernel = reconstruct_gptq_3bit_kernel;
1481 + gridDim.y = DIVIDE(height, 32);
1482 + }
1483 +
1484 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1485 + kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,
1486 + b_gptq_qzeros, b_g_idx, height,
1487 + width, groups, out);
1488 }
1489
1633 -
1634 -void gemm_half_q_half_cuda
1635 -(
1636 - cublasHandle_t cublas_handle,
1637 - const half* a,
1638 - const uint32_t* b_q_weight,
1639 - const uint32_t* b_gptq_qzeros,
1640 - const half* b_gptq_scales,
1641 - const int* b_g_idx,
1642 - half* c,
1643 - half* temp_dq,
1644 - int size_m,
1645 - int size_n,
1646 - int size_k,
1647 - int groups,
1648 - bool use_exllama,
1649 - int bit
1650 -)
1651 -{
1652 - bool use_reconstruct;
1490 +void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
1491 + const uint32_t* b_q_weight,
1492 + const uint32_t* b_gptq_qzeros,
1493 + const half* b_gptq_scales, const int* b_g_idx,
1494 + half* c, half* temp_dq, int size_m, int size_n,
1495 + int size_k, int groups, bool use_exllama, int bit) {
1496 + bool use_reconstruct;
1497 + if (use_exllama) {
1498 + use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||
1499 + (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1500 + } else {
1501 + // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so
1502 + // we disabled them for now.
1503 + use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1504 + }
1505 + if (use_reconstruct) {
1506 + // Reconstruct FP16 matrix, then cuBLAS
1507 if (use_exllama) {
1654 - use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1508 + reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1509 + temp_dq, size_k, size_n, groups, bit);
1510 } else {
1656 - // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so we disabled them for now.
1657 - use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1511 + reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1512 + temp_dq, size_k, size_n, groups, bit);
1513 }
1659 - if (use_reconstruct) {
1660 - // Reconstruct FP16 matrix, then cuBLAS
1661 - if (use_exllama) {
1662 - reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
1663 - size_k, size_n, groups, bit);
1664 - }
1665 - else
1666 - {
1667 - reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1668 - temp_dq, size_k, size_n, groups, bit);
1669 - }
1514
1671 - const half alpha = __float2half(1.0f);
1672 - const half beta = __float2half(0.0f);
1673 - cublasHgemm(cublas_handle,
1674 - CUBLAS_OP_N,
1675 - CUBLAS_OP_N,
1676 - size_n, size_m, size_k,
1677 - &alpha, temp_dq, size_n,
1678 - a, size_k,
1679 - &beta, c, size_n);
1515 + const half alpha = __float2half(1.0f);
1516 + const half beta = __float2half(0.0f);
1517 + cublasHgemm(cublas_handle, CUBLAS_OP_N, CUBLAS_OP_N, size_n, size_m, size_k,
1518 + &alpha, temp_dq, size_n, a, size_k, &beta, c, size_n);
1519 + } else if (use_exllama) {
1520 + // Quantized matmul
1521 + int max_chunks = size_m / BLOCK_M_SIZE_MAX;
1522 + int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
1523 + int last_chunk_size = size_m - last_chunk;
1524 +
1525 + if (max_chunks) {
1526 + gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
1527 + b_g_idx, c, last_chunk, size_n, size_k,
1528 + BLOCK_M_SIZE_MAX, groups, bit);
1529 }
1681 - else if (use_exllama)
1682 - {
1683 - // Quantized matmul
1684 - int max_chunks = size_m / BLOCK_M_SIZE_MAX;
1685 - int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
1686 - int last_chunk_size = size_m - last_chunk;
1687 -
1688 - if (max_chunks)
1689 - {
1690 - gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1691 - c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
1692 - groups, bit);
1693 - }
1530
1695 - if (last_chunk_size)
1696 - {
1697 - gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
1698 - b_gptq_scales, b_g_idx, c + last_chunk * size_n,
1699 - last_chunk_size, size_n, size_k, last_chunk_size,
1700 - groups, bit);
1701 - }
1702 - }
1703 - else
1704 - {
1705 - gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1706 - c, size_m, size_n, size_k, bit);
1531 + if (last_chunk_size) {
1532 + gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight,
1533 + b_gptq_qzeros, b_gptq_scales, b_g_idx,
1534 + c + last_chunk * size_n, last_chunk_size,
1535 + size_n, size_k, last_chunk_size, groups, bit);
1536 }
1537 + } else {
1538 + gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1539 + c, size_m, size_n, size_k, bit);
1540 + }
1541 }
1542
1710 -__global__ void shuffle_4bit_kernel
1711 -(
1712 - uint32_t* __restrict__ b_q_weight,
1713 - const int size_k,
1714 - const int size_n
1715 -)
1716 -{
1717 - int n = blockIdx.x * THREADS_X + threadIdx.x;
1718 - if (n >= size_n) return;
1719 - int k = 0;
1720 - uint32_t* b_ptr = b_q_weight + n;
1721 - while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
1543 +__global__ void shuffle_4bit_kernel(uint32_t* __restrict__ b_q_weight,
1544 + const int size_k, const int size_n) {
1545 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1546 + if (n >= size_n) return;
1547 + int k = 0;
1548 + uint32_t* b_ptr = b_q_weight + n;
1549 + while (k < size_k) {
1550 + shuffle_4bit_8(b_ptr, size_n);
1551 + b_ptr += 1 * size_n;
1552 + k += 8;
1553 + }
1554 }
1555
1724 -__global__ void shuffle_8bit_kernel
1725 -(
1726 - uint32_t* __restrict__ b_q_weight,
1727 - const int size_k,
1728 - const int size_n
1729 -)
1730 -{
1731 - int n = blockIdx.x * THREADS_X + threadIdx.x;
1732 - if (n >= size_n) return;
1733 - int k = 0;
1734 - uint32_t* b_ptr = b_q_weight + n;
1735 - while (k < size_k) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }
1556 +__global__ void shuffle_8bit_kernel(uint32_t* __restrict__ b_q_weight,
1557 + const int size_k, const int size_n) {
1558 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1559 + if (n >= size_n) return;
1560 + int k = 0;
1561 + uint32_t* b_ptr = b_q_weight + n;
1562 + while (k < size_k) {
1563 + shuffle_8bit_4(b_ptr, size_n);
1564 + b_ptr += 1 * size_n;
1565 + k += 4;
1566 + }
1567 }
1568
1738 -__global__ void shuffle_2bit_kernel
1739 -(
1740 - uint32_t* __restrict__ b_q_weight,
1741 - const int size_k,
1742 - const int size_n
1743 -)
1744 -{
1745 - int n = blockIdx.x * THREADS_X + threadIdx.x;
1746 - if (n >= size_n) return;
1747 - int k = 0;
1748 - uint32_t* b_ptr = b_q_weight + n;
1749 - while (k < size_k) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }
1569 +__global__ void shuffle_2bit_kernel(uint32_t* __restrict__ b_q_weight,
1570 + const int size_k, const int size_n) {
1571 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1572 + if (n >= size_n) return;
1573 + int k = 0;
1574 + uint32_t* b_ptr = b_q_weight + n;
1575 + while (k < size_k) {
1576 + shuffle_2bit_16(b_ptr, size_n);
1577 + b_ptr += 1 * size_n;
1578 + k += 16;
1579 + }
1580 }
1581
1752 -__global__ void shuffle_3bit_kernel
1753 -(
1754 - uint32_t* __restrict__ b_q_weight,
1755 - const int size_k,
1756 - const int size_n
1757 -)
1758 -{
1759 - int n = blockIdx.x * THREADS_X + threadIdx.x;
1760 - if (n >= size_n) return;
1761 - int k = 0;
1762 - uint32_t* b_ptr = b_q_weight + n;
1763 - while (k < size_k) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }
1582 +__global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight,
1583 + const int size_k, const int size_n) {
1584 + int n = blockIdx.x * THREADS_X + threadIdx.x;
1585 + if (n >= size_n) return;
1586 + int k = 0;
1587 + uint32_t* b_ptr = b_q_weight + n;
1588 + while (k < size_k) {
1589 + shuffle_3bit_32(b_ptr, size_n);
1590 + b_ptr += 3 * size_n;
1591 + k += 32;
1592 + }
1593 }
1594
1766 -__global__ void make_sequential_4bit_kernel
1767 -(
1768 - const uint32_t* __restrict__ w,
1769 - uint32_t* __restrict__ w_new,
1770 - const int* __restrict__ q_perm,
1771 - const int w_width
1772 -)
1773 -{
1774 - const uint64_t* w2 = (uint64_t*) w;
1775 - uint64_t* w_new2 = (uint64_t*) w_new;
1776 - int w2_stride = w_width >> 1;
1777 - int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1778 - if (w2_column >= w2_stride) return;
1779 - int w_new2_row = blockIdx.y;
1780 - int q_perm_idx = w_new2_row << 3;
1781 - uint64_t dst = 0;
1782 -
1783 - #pragma unroll
1784 - for (int i = 0; i < 8; i++)
1785 - {
1786 - int source_row = q_perm[q_perm_idx++];
1787 -
1788 - int w2_row = source_row >> 3;
1789 - int w2_subrow = source_row & 0x07;
1790 - int w2_row_shift = w2_subrow << 2;
1791 - int wnew2_row_shift = i << 2;
1792 -
1793 - uint64_t src = w2[w2_row * w2_stride + w2_column];
1794 - src >>= w2_row_shift;
1795 - src &= 0x0000000f0000000f;
1796 - src <<= wnew2_row_shift;
1797 - dst |= src;
1798 - }
1799 - w_new2[w_new2_row * w2_stride + w2_column] = dst;
1595 +__global__ void make_sequential_4bit_kernel(const uint32_t* __restrict__ w,
1596 + uint32_t* __restrict__ w_new,
1597 + const int* __restrict__ q_perm,
1598 + const int w_width) {
1599 + const uint64_t* w2 = (uint64_t*)w;
1600 + uint64_t* w_new2 = (uint64_t*)w_new;
1601 + int w2_stride = w_width >> 1;
1602 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1603 + if (w2_column >= w2_stride) return;
1604 + int w_new2_row = blockIdx.y;
1605 + int q_perm_idx = w_new2_row << 3;
1606 + uint64_t dst = 0;
1607 +
1608 +#pragma unroll
1609 + for (int i = 0; i < 8; i++) {
1610 + int source_row = q_perm[q_perm_idx++];
1611 +
1612 + int w2_row = source_row >> 3;
1613 + int w2_subrow = source_row & 0x07;
1614 + int w2_row_shift = w2_subrow << 2;
1615 + int wnew2_row_shift = i << 2;
1616 +
1617 + uint64_t src = w2[w2_row * w2_stride + w2_column];
1618 + src >>= w2_row_shift;
1619 + src &= 0x0000000f0000000f;
1620 + src <<= wnew2_row_shift;
1621 + dst |= src;
1622 + }
1623 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
1624 }
1625
1802 -__global__ void make_sequential_2bit_kernel
1803 -(
1804 - const uint32_t* __restrict__ w,
1805 - uint32_t* __restrict__ w_new,
1806 - const int* __restrict__ q_perm,
1807 - const int w_width
1808 -)
1809 -{
1810 - const uint64_t* w2 = (uint64_t*) w;
1811 - uint64_t* w_new2 = (uint64_t*) w_new;
1812 - int w2_stride = w_width >> 1;
1813 - int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1814 - if (w2_column >= w2_stride) return;
1815 - int w_new2_row = blockIdx.y;
1816 - int q_perm_idx = w_new2_row << 4;
1817 - uint64_t dst = 0;
1818 -
1819 - #pragma unroll
1820 - for (int i = 0; i < 16; i++)
1821 - {
1822 - int source_row = q_perm[q_perm_idx++];
1823 -
1824 - int w2_row = source_row >> 4;
1825 - int w2_subrow = source_row & 0x0f;
1826 - int w2_row_shift = w2_subrow << 1;
1827 - int wnew2_row_shift = i << 1;
1828 -
1829 - uint64_t src = w2[w2_row * w2_stride + w2_column];
1830 - src >>= w2_row_shift;
1831 - src &= 0x0000000300000003;
1832 - src <<= wnew2_row_shift;
1833 - dst |= src;
1834 - }
1835 - w_new2[w_new2_row * w2_stride + w2_column] = dst;
1626 +__global__ void make_sequential_2bit_kernel(const uint32_t* __restrict__ w,
1627 + uint32_t* __restrict__ w_new,
1628 + const int* __restrict__ q_perm,
1629 + const int w_width) {
1630 + const uint64_t* w2 = (uint64_t*)w;
1631 + uint64_t* w_new2 = (uint64_t*)w_new;
1632 + int w2_stride = w_width >> 1;
1633 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1634 + if (w2_column >= w2_stride) return;
1635 + int w_new2_row = blockIdx.y;
1636 + int q_perm_idx = w_new2_row << 4;
1637 + uint64_t dst = 0;
1638 +
1639 +#pragma unroll
1640 + for (int i = 0; i < 16; i++) {
1641 + int source_row = q_perm[q_perm_idx++];
1642 +
1643 + int w2_row = source_row >> 4;
1644 + int w2_subrow = source_row & 0x0f;
1645 + int w2_row_shift = w2_subrow << 1;
1646 + int wnew2_row_shift = i << 1;
1647 +
1648 + uint64_t src = w2[w2_row * w2_stride + w2_column];
1649 + src >>= w2_row_shift;
1650 + src &= 0x0000000300000003;
1651 + src <<= wnew2_row_shift;
1652 + dst |= src;
1653 + }
1654 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
1655 }
1656
1838 -__global__ void make_sequential_3bit_kernel
1839 -(
1840 - const uint32_t* __restrict__ w,
1841 - uint32_t* __restrict__ w_new,
1842 - const int* __restrict__ q_perm,
1843 - const int w_width
1844 -)
1845 -{
1846 - int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1847 - if (w_column >= w_width) return;
1848 - int w_new_row = blockIdx.y * 3;
1849 - int q_perm_idx = blockIdx.y << 5;
1850 - uint32_t dst[3] = {0, 0, 0};
1851 -
1852 - #pragma unroll
1853 - for (int i = 0; i < 32; i++)
1854 - {
1855 - int source_row = q_perm[q_perm_idx++];
1856 - int z_w = (source_row / 32) * 3;
1857 - int z_mod = source_row % 32;
1858 - int z_bit;
1859 -
1860 - if (z_mod != 10){
1861 - if (z_mod != 21){
1862 - z_bit = z_mod;
1863 - if (z_bit > 21){
1864 - z_bit *= 3;
1865 - z_bit -= 64;
1866 - z_w += 2;
1867 - } else if (z_bit > 10){
1868 - z_bit *= 3;
1869 - z_bit -= 32;
1870 - z_w += 1;
1871 - } else {
1872 - z_bit *= 3;
1873 - }
1874 - } else {
1875 - z_w += 1;
1876 - }
1877 - }
1878 -
1879 - uint64_t src;
1880 - if (z_mod == 10) {
1881 - src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1882 - } else if (z_mod == 21){
1883 - src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1657 +__global__ void make_sequential_3bit_kernel(const uint32_t* __restrict__ w,
1658 + uint32_t* __restrict__ w_new,
1659 + const int* __restrict__ q_perm,
1660 + const int w_width) {
1661 + int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1662 + if (w_column >= w_width) return;
1663 + int w_new_row = blockIdx.y * 3;
1664 + int q_perm_idx = blockIdx.y << 5;
1665 + uint32_t dst[3] = {0, 0, 0};
1666 +
1667 +#pragma unroll
1668 + for (int i = 0; i < 32; i++) {
1669 + int source_row = q_perm[q_perm_idx++];
1670 + int z_w = (source_row / 32) * 3;
1671 + int z_mod = source_row % 32;
1672 + int z_bit;
1673 +
1674 + if (z_mod != 10) {
1675 + if (z_mod != 21) {
1676 + z_bit = z_mod;
1677 + if (z_bit > 21) {
1678 + z_bit *= 3;
1679 + z_bit -= 64;
1680 + z_w += 2;
1681 + } else if (z_bit > 10) {
1682 + z_bit *= 3;
1683 + z_bit -= 32;
1684 + z_w += 1;
1685 } else {
1885 - src = w[z_w * w_width + w_column];
1886 - src >>= z_bit;
1887 - src &= 0x07;
1686 + z_bit *= 3;
1687 }
1688 + } else {
1689 + z_w += 1;
1690 + }
1691 + }
1692
1890 - z_w = 0;
1891 - if (i != 10){
1892 - if (i != 21){
1893 - z_bit = i;
1894 - if (z_bit > 21){
1895 - z_bit *= 3;
1896 - z_bit -= 64;
1897 - z_w += 2;
1898 - } else if (z_bit > 10){
1899 - z_bit *= 3;
1900 - z_bit -= 32;
1901 - z_w += 1;
1902 - } else {
1903 - z_bit *= 3;
1904 - }
1905 - } else {
1906 - z_w += 1;
1907 - }
1908 - }
1909 - if (i == 10) {
1910 - dst[z_w] |= (src & 0x03) << 30;
1911 - dst[z_w + 1] |= ((src & 0x4) >> 2);
1912 - } else if (i == 21) {
1913 - dst[z_w] |= (src & 0x01) << 31;
1914 - dst[z_w + 1] |= ((src & 0x6) >> 1);
1693 + uint64_t src;
1694 + if (z_mod == 10) {
1695 + src = (w[z_w * w_width + w_column] >> 30) |
1696 + ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1697 + } else if (z_mod == 21) {
1698 + src = (w[z_w * w_width + w_column] >> 31) |
1699 + ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1700 + } else {
1701 + src = w[z_w * w_width + w_column];
1702 + src >>= z_bit;
1703 + src &= 0x07;
1704 + }
1705 +
1706 + z_w = 0;
1707 + if (i != 10) {
1708 + if (i != 21) {
1709 + z_bit = i;
1710 + if (z_bit > 21) {
1711 + z_bit *= 3;
1712 + z_bit -= 64;
1713 + z_w += 2;
1714 + } else if (z_bit > 10) {
1715 + z_bit *= 3;
1716 + z_bit -= 32;
1717 + z_w += 1;
1718 } else {
1916 - dst[z_w] |= (src << z_bit);
1719 + z_bit *= 3;
1720 }
1721 + } else {
1722 + z_w += 1;
1723 + }
1724 + }
1725 + if (i == 10) {
1726 + dst[z_w] |= (src & 0x03) << 30;
1727 + dst[z_w + 1] |= ((src & 0x4) >> 2);
1728 + } else if (i == 21) {
1729 + dst[z_w] |= (src & 0x01) << 31;
1730 + dst[z_w + 1] |= ((src & 0x6) >> 1);
1731 + } else {
1732 + dst[z_w] |= (src << z_bit);
1733 }
1919 - w_new[w_new_row * w_width + w_column] = dst[0];
1920 - w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1921 - w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1734 + }
1735 + w_new[w_new_row * w_width + w_column] = dst[0];
1736 + w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1737 + w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1738 }
1739
1924 -__global__ void make_sequential_8bit_kernel
1925 -(
1926 - const uint32_t* __restrict__ w,
1927 - uint32_t* __restrict__ w_new,
1928 - const int* __restrict__ q_perm,
1929 - const int w_width
1930 -)
1931 -{
1932 - const uint64_t* w2 = (uint64_t*) w;
1933 - uint64_t* w_new2 = (uint64_t*) w_new;
1934 - int w2_stride = w_width >> 1;
1935 - int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1936 - if (w2_column >= w2_stride) return;
1937 - int w_new2_row = blockIdx.y;
1938 - int q_perm_idx = w_new2_row << 2;
1939 - uint64_t dst = 0;
1940 -
1941 - #pragma unroll
1942 - for (int i = 0; i < 4; i++)
1943 - {
1944 - int source_row = q_perm[q_perm_idx++];
1945 -
1946 - int w2_row = source_row >> 2;
1947 - int w2_subrow = source_row & 0x03;
1948 - int w2_row_shift = w2_subrow << 3;
1949 - int wnew2_row_shift = i << 3;
1950 -
1951 - uint64_t src = w2[w2_row * w2_stride + w2_column];
1952 - src >>= w2_row_shift;
1953 - src &= 0x000000ff000000ff;
1954 - src <<= wnew2_row_shift;
1955 - dst |= src;
1956 - }
1957 - w_new2[w_new2_row * w2_stride + w2_column] = dst;
1740 +__global__ void make_sequential_8bit_kernel(const uint32_t* __restrict__ w,
1741 + uint32_t* __restrict__ w_new,
1742 + const int* __restrict__ q_perm,
1743 + const int w_width) {
1744 + const uint64_t* w2 = (uint64_t*)w;
1745 + uint64_t* w_new2 = (uint64_t*)w_new;
1746 + int w2_stride = w_width >> 1;
1747 + int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1748 + if (w2_column >= w2_stride) return;
1749 + int w_new2_row = blockIdx.y;
1750 + int q_perm_idx = w_new2_row << 2;
1751 + uint64_t dst = 0;
1752 +
1753 +#pragma unroll
1754 + for (int i = 0; i < 4; i++) {
1755 + int source_row = q_perm[q_perm_idx++];
1756 +
1757 + int w2_row = source_row >> 2;
1758 + int w2_subrow = source_row & 0x03;
1759 + int w2_row_shift = w2_subrow << 3;
1760 + int wnew2_row_shift = i << 3;
1761 +
1762 + uint64_t src = w2[w2_row * w2_stride + w2_column];
1763 + src >>= w2_row_shift;
1764 + src &= 0x000000ff000000ff;
1765 + src <<= wnew2_row_shift;
1766 + dst |= src;
1767 + }
1768 + w_new2[w_new2_row * w2_stride + w2_column] = dst;
1769 }
1770
1771 +void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
1772 + int width, int bit) {
1773 + if (q_perm) {
1774 + uint32_t* new_qweight = NULL;
1775 + cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1776
1961 -void shuffle_exllama_weight
1962 -(
1963 - uint32_t* q_weight,
1964 - int* q_perm,
1965 - int height,
1966 - int width,
1967 - int bit
1968 -)
1969 -{
1970 - if (q_perm)
1971 - {
1972 - uint32_t* new_qweight = NULL;
1973 - cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1974 -
1975 - dim3 blockDim, gridDim;
1976 - blockDim.x = THREADS_X;
1977 - blockDim.y = 1;
1978 - gridDim.x = DIVIDE(width, THREADS_X);
1979 - gridDim.y = height / 32 * bit;
1980 -
1981 - auto kernel = make_sequential_4bit_kernel;
1982 - if (bit == 2) {
1983 - kernel = make_sequential_2bit_kernel;
1984 - } else if (bit == 3) {
1985 - kernel = make_sequential_3bit_kernel;
1986 - gridDim.y = height / 32;
1987 - } else if (bit == 8) {
1988 - kernel = make_sequential_8bit_kernel;
1989 - }
1990 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1991 - kernel<<<gridDim, blockDim, 0, stream>>>
1992 - (
1993 - q_weight,
1994 - new_qweight,
1995 - q_perm,
1996 - width
1997 - );
1998 - // Replace qweights
1999 - cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
2000 - // Cleanup
2001 - cudaDeviceSynchronize();
2002 - cudaFree(new_qweight);
2003 - }
1777 dim3 blockDim, gridDim;
1778 blockDim.x = THREADS_X;
1779 blockDim.y = 1;
1780 gridDim.x = DIVIDE(width, THREADS_X);
2008 - gridDim.y = 1;
2009 - auto shuffle_kernel = shuffle_4bit_kernel;
1781 + gridDim.y = height / 32 * bit;
1782 +
1783 + auto kernel = make_sequential_4bit_kernel;
1784 if (bit == 2) {
2011 - shuffle_kernel = shuffle_2bit_kernel;
1785 + kernel = make_sequential_2bit_kernel;
1786 } else if (bit == 3) {
2013 - shuffle_kernel = shuffle_3bit_kernel;
1787 + kernel = make_sequential_3bit_kernel;
1788 + gridDim.y = height / 32;
1789 } else if (bit == 8) {
2015 - shuffle_kernel = shuffle_8bit_kernel;
1790 + kernel = make_sequential_8bit_kernel;
1791 }
1792 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
2018 - shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
1793 + kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, new_qweight, q_perm,
1794 + width);
1795 + // Replace qweights
1796 + cudaMemcpyAsync(q_weight, new_qweight,
1797 + height / 32 * bit * width * sizeof(uint32_t),
1798 + cudaMemcpyDeviceToDevice);
1799 + // Cleanup
1800 + cudaDeviceSynchronize();
1801 + cudaFree(new_qweight);
1802 + }
1803 + dim3 blockDim, gridDim;
1804 + blockDim.x = THREADS_X;
1805 + blockDim.y = 1;
1806 + gridDim.x = DIVIDE(width, THREADS_X);
1807 + gridDim.y = 1;
1808 + auto shuffle_kernel = shuffle_4bit_kernel;
1809 + if (bit == 2) {
1810 + shuffle_kernel = shuffle_2bit_kernel;
1811 + } else if (bit == 3) {
1812 + shuffle_kernel = shuffle_3bit_kernel;
1813 + } else if (bit == 8) {
1814 + shuffle_kernel = shuffle_8bit_kernel;
1815 + }
1816 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1817 + shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
1818 }
1819
1820 } // namespace gptq
1821 } // namespace vllm
1822
2024 -torch::Tensor gptq_gemm
2025 -(
2026 - torch::Tensor a,
2027 - torch::Tensor b_q_weight,
2028 - torch::Tensor b_gptq_qzeros,
2029 - torch::Tensor b_gptq_scales,
2030 - torch::Tensor b_g_idx,
2031 - bool use_exllama,
2032 - int bit
2033 -)
2034 -{
2035 - const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
2036 - auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
2037 - at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
2038 - at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
2039 -
2040 - vllm::gptq::gemm_half_q_half_cuda
2041 - (
2042 - at::cuda::getCurrentCUDABlasHandle(),
2043 - (const half*) a.data_ptr(),
2044 - (const uint32_t*) b_q_weight.data_ptr(),
2045 - (const uint32_t*)b_gptq_qzeros.data_ptr(),
2046 - (const half*) b_gptq_scales.data_ptr(),
2047 - b_g_idx.device().is_meta() ? NULL : (const int*) b_g_idx.data_ptr(),
2048 - (half*) c.data_ptr(),
2049 - (half*) temp_dq.data_ptr(),
2050 - c.size(0), // m
2051 - c.size(1), // n
2052 - a.size(1), // k
2053 - b_gptq_qzeros.size(0), // group number
2054 - use_exllama,
2055 - bit
2056 - );
2057 - return c;
1823 +torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1824 + torch::Tensor b_gptq_qzeros,
1825 + torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
1826 + bool use_exllama, int bit) {
1827 + const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1828 + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1829 + at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
1830 + at::Tensor temp_dq = torch::empty(
1831 + {b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
1832 +
1833 + vllm::gptq::gemm_half_q_half_cuda(
1834 + at::cuda::getCurrentCUDABlasHandle(), (const half*)a.data_ptr(),
1835 + (const uint32_t*)b_q_weight.data_ptr(),
1836 + (const uint32_t*)b_gptq_qzeros.data_ptr(),
1837 + (const half*)b_gptq_scales.data_ptr(),
1838 + b_g_idx.device().is_meta() ? NULL : (const int*)b_g_idx.data_ptr(),
1839 + (half*)c.data_ptr(), (half*)temp_dq.data_ptr(),
1840 + c.size(0), // m
1841 + c.size(1), // n
1842 + a.size(1), // k
1843 + b_gptq_qzeros.size(0), // group number
1844 + use_exllama, bit);
1845 + return c;
1846 }
1847
2060 -void gptq_shuffle
2061 -(
2062 - torch::Tensor q_weight,
2063 - torch::Tensor q_perm,
2064 - int bit
2065 -)
2066 -{
2067 - const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068 - vllm::gptq::shuffle_exllama_weight(
2069 - (uint32_t*) q_weight.data_ptr(),
2070 - q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*) q_perm.data_ptr(),
2071 - q_weight.size(0) * 32 / bit,
2072 - q_weight.size(1),
2073 - bit
2074 - );
1848 +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit) {
1849 + const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
1850 + vllm::gptq::shuffle_exllama_weight(
1851 + (uint32_t*)q_weight.data_ptr(),
1852 + q_perm.device().is_meta() || q_perm.numel() == 0
1853 + ? NULL
1854 + : (int*)q_perm.data_ptr(),
1855 + q_weight.size(0) * 32 / bit, q_weight.size(1), bit);
1856 }
csrc/quantization/gptq/qdq_2.cuh
+48 -59
@@ -14,71 +14,60 @@ namespace gptq {
14 //
15 // ffddbb99 77553311 eeccaa88 66442200
16
17 -__forceinline__ __device__ void shuffle_2bit_16
18 -(
19 - uint32_t* q,
20 - int stride
21 -)
22 -{
23 - uint32_t qa = q[0];
24 - uint32_t qb = 0;
17 +__forceinline__ __device__ void shuffle_2bit_16(uint32_t* q, int stride) {
18 + uint32_t qa = q[0];
19 + uint32_t qb = 0;
20
26 - #pragma unroll
27 - for (int i = 0; i < 8; i++)
28 - {
29 - uint32_t qa0 = qa & 0x03;
30 - uint32_t qa1 = (qa & 0x0c) >> 2;
31 - qa >>= 4;
32 - qb |= (qa1 << (i * 2 + 16));
33 - qb |= (qa0 << (i * 2));
34 - }
35 - q[0] = qb;
21 +#pragma unroll
22 + for (int i = 0; i < 8; i++) {
23 + uint32_t qa0 = qa & 0x03;
24 + uint32_t qa1 = (qa & 0x0c) >> 2;
25 + qa >>= 4;
26 + qb |= (qa1 << (i * 2 + 16));
27 + qb |= (qa0 << (i * 2));
28 + }
29 + q[0] = qb;
30 }
31
38 -__forceinline__ __device__ void dequant_2bit_16
39 -(
40 - const uint32_t q_0,
41 - half2 (&dq)[8],
42 - int stride,
43 - const uint32_t zero
44 -)
45 -{
46 - const uint32_t c0 = 0x64006400;
47 - const half y4_ = __float2half_rn(1.0f / 4.0f);
48 - const half y16_ = __float2half_rn(1.0f / 16.0f);
49 - const half y64_ = __float2half_rn(1.0f / 64.0f);
50 - const half2 y4 = __halves2half2(y4_, y4_);
51 - const half2 y16 = __halves2half2(y16_, y16_);
52 - const half2 y64 = __halves2half2(y64_, y64_);
32 +__forceinline__ __device__ void dequant_2bit_16(const uint32_t q_0,
33 + half2 (&dq)[8], int stride,
34 + const uint32_t zero) {
35 + const uint32_t c0 = 0x64006400;
36 + const half y4_ = __float2half_rn(1.0f / 4.0f);
37 + const half y16_ = __float2half_rn(1.0f / 16.0f);
38 + const half y64_ = __float2half_rn(1.0f / 64.0f);
39 + const half2 y4 = __halves2half2(y4_, y4_);
40 + const half2 y16 = __halves2half2(y16_, y16_);
41 + const half2 y64 = __halves2half2(y64_, y64_);
42
54 - const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
55 - const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
56 - const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
57 - const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
58 - const half2 z1 = __half2half2(z1_.as_half);
59 - const half2 z4 = __half2half2(z4_);
60 - const half2 z16 = __half2half2(z16_);
61 - const half2 z64 = __half2half2(z64_);
43 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
44 + const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
45 + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
46 + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
47 + const half2 z1 = __half2half2(z1_.as_half);
48 + const half2 z4 = __half2half2(z4_);
49 + const half2 z16 = __half2half2(z16_);
50 + const half2 z64 = __half2half2(z64_);
51
63 - uint32_t qa = q_0;
64 - half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
65 - half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
66 - half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
67 - half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
68 - qa >>= 8;
69 - half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
70 - half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
71 - half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
72 - half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
52 + uint32_t qa = q_0;
53 + half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
54 + half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
55 + half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
56 + half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
57 + qa >>= 8;
58 + half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
59 + half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
60 + half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
61 + half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
62
74 - dq[0] = __hadd2(q0.as_half2, z1);
75 - dq[1] = __hfma2(q1.as_half2, y4, z4);
76 - dq[2] = __hfma2(q2.as_half2, y16, z16);
77 - dq[3] = __hfma2(q3.as_half2, y64, z64);
78 - dq[4] = __hadd2(q4.as_half2, z1);
79 - dq[5] = __hfma2(q5.as_half2, y4, z4);
80 - dq[6] = __hfma2(q6.as_half2, y16, z16);
81 - dq[7] = __hfma2(q7.as_half2, y64, z64);
63 + dq[0] = __hadd2(q0.as_half2, z1);
64 + dq[1] = __hfma2(q1.as_half2, y4, z4);
65 + dq[2] = __hfma2(q2.as_half2, y16, z16);
66 + dq[3] = __hfma2(q3.as_half2, y64, z64);
67 + dq[4] = __hadd2(q4.as_half2, z1);
68 + dq[5] = __hfma2(q5.as_half2, y4, z4);
69 + dq[6] = __hfma2(q6.as_half2, y16, z16);
70 + dq[7] = __hfma2(q7.as_half2, y64, z64);
71 }
72
73 } // namespace gptq
csrc/quantization/gptq/qdq_3.cuh
+127 -119
@@ -11,128 +11,136 @@ namespace gptq {
11 // vjjjhhhf ffdddbbb uiiiggge eecccaaa
12 // vtttrrrp ppnnnlll usssqqqo oommmkkk
13
14 -__forceinline__ __device__ void shuffle_3bit_32
15 -(
16 - uint32_t* q,
17 - int stride
18 -)
19 -{
20 - uint32_t qa = q[0 * stride];
21 - uint32_t qb = q[1 * stride];
22 - uint32_t qc = q[2 * stride];
23 -
24 - // qa: aa999888 77766655 54443332 22111000
25 - // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
26 - // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
27 -
28 - uint32_t qd = qc >> 26;
29 - qc <<= 4;
30 - qc |= qb >> 28;
31 - qb <<= 2;
32 - qb |= qa >> 30;
33 -
34 - // qa: ..999888 77766655 54443332 22111000
35 - // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
36 - // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
37 - // qd: vvvuuu
38 -
39 - uint32_t za = 0;
40 - uint32_t zb = 0;
41 - uint32_t zc = 0;
42 -
43 - for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }
44 - for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }
45 - for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }
46 -
47 - // za: 9997775 55333111 8886664 44222000
48 - // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
49 - // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
50 - // qd: vvvuuu
51 -
52 - za |= ((qd & 0x01) >> 0) << 15;
53 - zb |= ((qd & 0x02) >> 1) << 15;
54 - zc |= ((qd & 0x04) >> 2) << 15;
55 - za |= ((qd & 0x08) >> 3) << 31;
56 - zb |= ((qd & 0x10) >> 4) << 31;
57 - zc |= ((qd & 0x20) >> 5) << 31;
58 -
59 - // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
60 - // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
61 - // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
62 -
63 - q[0 * stride] = za;
64 - q[1 * stride] = zb;
65 - q[2 * stride] = zc;
66 -}
67 -
68 -__forceinline__ __device__ void dequant_3bit_32
69 -(
70 - const uint32_t q_0,
71 - const uint32_t q_1,
72 - const uint32_t q_2,
73 - half2 (&dq)[16],
74 - int stride,
75 - const uint32_t zero
76 -)
77 -{
78 - const uint32_t c0 = 0x64006400;
79 - const half y8_ = __float2half_rn(1.0f / 8.0f);
80 - const half y64_ = __float2half_rn(1.0f / 64.0f);
81 - const half2 y8 = __halves2half2(y8_, y8_);
82 - const half2 y64 = __halves2half2(y64_, y64_);
83 - const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
84 - const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
85 - const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
86 - const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
87 - const half2 z8 = __halves2half2(z8_, z8_);
88 - const half2 z64 = __halves2half2(z64_, z64_);
89 -
90 - uint32_t qa = q_0;
91 - uint32_t qb = q_1;
92 - uint32_t qc = q_2;
93 -
94 - half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
95 - half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
14 +__forceinline__ __device__ void shuffle_3bit_32(uint32_t* q, int stride) {
15 + uint32_t qa = q[0 * stride];
16 + uint32_t qb = q[1 * stride];
17 + uint32_t qc = q[2 * stride];
18 +
19 + // qa: aa999888 77766655 54443332 22111000
20 + // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
21 + // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
22 +
23 + uint32_t qd = qc >> 26;
24 + qc <<= 4;
25 + qc |= qb >> 28;
26 + qb <<= 2;
27 + qb |= qa >> 30;
28 +
29 + // qa: ..999888 77766655 54443332 22111000
30 + // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
31 + // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
32 + // qd: vvvuuu
33 +
34 + uint32_t za = 0;
35 + uint32_t zb = 0;
36 + uint32_t zc = 0;
37 +
38 + for (int i = 0; i < 5; i++) {
39 + uint32_t t0 = qa & 0x07;
40 + uint32_t t1 = (qa & 0x38) >> 3;
41 qa >>= 6;
97 - half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
98 - half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
99 - half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
100 - qa >>= 9;
101 - qa &= 0x00010001;
102 - half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
103 - half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
42 + za |= (t0 << (i * 3));
43 + za |= (t1 << (i * 3 + 16));
44 + }
45 + for (int i = 0; i < 5; i++) {
46 + uint32_t t0 = qb & 0x07;
47 + uint32_t t1 = (qb & 0x38) >> 3;
48 qb >>= 6;
105 - half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
106 - half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
107 - half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
108 - qb >>= 8;
109 - qb &= 0x00020002;
110 - half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
111 - half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
49 + zb |= (t0 << (i * 3));
50 + zb |= (t1 << (i * 3 + 16));
51 + }
52 + for (int i = 0; i < 5; i++) {
53 + uint32_t t0 = qc & 0x07;
54 + uint32_t t1 = (qc & 0x38) >> 3;
55 qc >>= 6;
113 - half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
114 - half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
115 - half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
116 - qc >>= 7;
117 - qc &= 0x00040004;
118 - half2_uint32 q15((qa | qb | qc) | c0);
119 -
120 - dq[ 0] = __hadd2( q0.as_half2, z1);
121 - dq[ 1] = __hfma2( q1.as_half2, y8, z8);
122 - dq[ 2] = __hadd2( q2.as_half2, z1);
123 - dq[ 3] = __hfma2( q3.as_half2, y8, z8);
124 - dq[ 4] = __hfma2( q4.as_half2, y64, z64);
125 - dq[ 5] = __hadd2( q5.as_half2, z1);
126 - dq[ 6] = __hfma2( q6.as_half2, y8, z8);
127 - dq[ 7] = __hadd2( q7.as_half2, z1);
128 - dq[ 8] = __hfma2( q8.as_half2, y8, z8);
129 - dq[ 9] = __hfma2( q9.as_half2, y64, z64);
130 - dq[10] = __hadd2(q10.as_half2, z1);
131 - dq[11] = __hfma2(q11.as_half2, y8, z8);
132 - dq[12] = __hadd2(q12.as_half2, z1);
133 - dq[13] = __hfma2(q13.as_half2, y8, z8);
134 - dq[14] = __hfma2(q14.as_half2, y64, z64);
135 - dq[15] = __hadd2(q15.as_half2, z1);
56 + zc |= (t0 << (i * 3));
57 + zc |= (t1 << (i * 3 + 16));
58 + }
59 +
60 + // za: 9997775 55333111 8886664 44222000
61 + // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
62 + // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
63 + // qd: vvvuuu
64 +
65 + za |= ((qd & 0x01) >> 0) << 15;
66 + zb |= ((qd & 0x02) >> 1) << 15;
67 + zc |= ((qd & 0x04) >> 2) << 15;
68 + za |= ((qd & 0x08) >> 3) << 31;
69 + zb |= ((qd & 0x10) >> 4) << 31;
70 + zc |= ((qd & 0x20) >> 5) << 31;
71 +
72 + // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
73 + // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
74 + // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
75 +
76 + q[0 * stride] = za;
77 + q[1 * stride] = zb;
78 + q[2 * stride] = zc;
79 +}
80 +
81 +__forceinline__ __device__ void dequant_3bit_32(const uint32_t q_0,
82 + const uint32_t q_1,
83 + const uint32_t q_2,
84 + half2 (&dq)[16], int stride,
85 + const uint32_t zero) {
86 + const uint32_t c0 = 0x64006400;
87 + const half y8_ = __float2half_rn(1.0f / 8.0f);
88 + const half y64_ = __float2half_rn(1.0f / 64.0f);
89 + const half2 y8 = __halves2half2(y8_, y8_);
90 + const half2 y64 = __halves2half2(y64_, y64_);
91 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
92 + const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
93 + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
94 + const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
95 + const half2 z8 = __halves2half2(z8_, z8_);
96 + const half2 z64 = __halves2half2(z64_, z64_);
97 +
98 + uint32_t qa = q_0;
99 + uint32_t qb = q_1;
100 + uint32_t qc = q_2;
101 +
102 + half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
103 + half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
104 + qa >>= 6;
105 + half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
106 + half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
107 + half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
108 + qa >>= 9;
109 + qa &= 0x00010001;
110 + half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
111 + half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
112 + qb >>= 6;
113 + half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
114 + half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
115 + half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
116 + qb >>= 8;
117 + qb &= 0x00020002;
118 + half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
119 + half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
120 + qc >>= 6;
121 + half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
122 + half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
123 + half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
124 + qc >>= 7;
125 + qc &= 0x00040004;
126 + half2_uint32 q15((qa | qb | qc) | c0);
127 +
128 + dq[0] = __hadd2(q0.as_half2, z1);
129 + dq[1] = __hfma2(q1.as_half2, y8, z8);
130 + dq[2] = __hadd2(q2.as_half2, z1);
131 + dq[3] = __hfma2(q3.as_half2, y8, z8);
132 + dq[4] = __hfma2(q4.as_half2, y64, z64);
133 + dq[5] = __hadd2(q5.as_half2, z1);
134 + dq[6] = __hfma2(q6.as_half2, y8, z8);
135 + dq[7] = __hadd2(q7.as_half2, z1);
136 + dq[8] = __hfma2(q8.as_half2, y8, z8);
137 + dq[9] = __hfma2(q9.as_half2, y64, z64);
138 + dq[10] = __hadd2(q10.as_half2, z1);
139 + dq[11] = __hfma2(q11.as_half2, y8, z8);
140 + dq[12] = __hadd2(q12.as_half2, z1);
141 + dq[13] = __hfma2(q13.as_half2, y8, z8);
142 + dq[14] = __hfma2(q14.as_half2, y64, z64);
143 + dq[15] = __hadd2(q15.as_half2, z1);
144 }
145
146 } // namespace gptq
csrc/quantization/gptq/qdq_4.cuh
+91 -112
@@ -13,133 +13,112 @@ namespace gptq {
13 //
14 // 77775555 33331111 66664444 22220000
15
16 -__forceinline__ __device__ void shuffle_4bit_8
17 -(
18 - uint32_t* q,
19 - int stride
20 -)
21 -{
22 - uint32_t qa = q[0];
23 - uint32_t qb = 0;
24 -
25 - #pragma unroll
26 - for (int i = 0; i < 4; i++)
27 - {
28 - uint32_t qa0 = qa & 0x0f;
29 - uint32_t qa1 = (qa & 0xf0) >> 4;
30 - qa >>= 8;
31 - qb |= (qa1 << (i * 4 + 16));
32 - qb |= (qa0 << (i * 4));
33 - }
34 - q[0] = qb;
35 -}
36 -
37 -__forceinline__ __device__ void dequant_4bit_8
38 -(
39 - const uint32_t q_0,
40 - half2 (&dq)[4],
41 - int stride,
42 - const uint32_t zero
43 -)
44 -{
45 - const uint32_t c0 = 0x64006400;
46 - const half y16_ = __float2half_rn(1.0f / 16.0f);
47 - const half2 y16 = __halves2half2(y16_, y16_);
48 - const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
49 - const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
50 - const half2 z1 = __half2half2(z1_.as_half);
51 - const half2 z16 = __half2half2(z16_);
52 -
53 - uint32_t qa = q_0;
54 - half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
55 - half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
16 +__forceinline__ __device__ void shuffle_4bit_8(uint32_t* q, int stride) {
17 + uint32_t qa = q[0];
18 + uint32_t qb = 0;
19 +
20 +#pragma unroll
21 + for (int i = 0; i < 4; i++) {
22 + uint32_t qa0 = qa & 0x0f;
23 + uint32_t qa1 = (qa & 0xf0) >> 4;
24 qa >>= 8;
57 - half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
58 - half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
25 + qb |= (qa1 << (i * 4 + 16));
26 + qb |= (qa0 << (i * 4));
27 + }
28 + q[0] = qb;
29 +}
30
60 - dq[0] = __hadd2(q0.as_half2, z1);
61 - dq[1] = __hfma2(q1.as_half2, y16, z16);
62 - dq[2] = __hadd2(q2.as_half2, z1);
63 - dq[3] = __hfma2(q3.as_half2, y16, z16);
31 +__forceinline__ __device__ void dequant_4bit_8(const uint32_t q_0,
32 + half2 (&dq)[4], int stride,
33 + const uint32_t zero) {
34 + const uint32_t c0 = 0x64006400;
35 + const half y16_ = __float2half_rn(1.0f / 16.0f);
36 + const half2 y16 = __halves2half2(y16_, y16_);
37 + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
38 + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
39 + const half2 z1 = __half2half2(z1_.as_half);
40 + const half2 z16 = __half2half2(z16_);
41 +
42 + uint32_t qa = q_0;
43 + half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
44 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
45 + qa >>= 8;
46 + half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
47 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
48 +
49 + dq[0] = __hadd2(q0.as_half2, z1);
50 + dq[1] = __hfma2(q1.as_half2, y16, z16);
51 + dq[2] = __hadd2(q2.as_half2, z1);
52 + dq[3] = __hfma2(q3.as_half2, y16, z16);
53 }
54
66 -__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
67 -(
68 - const uint32_t zero,
69 - const half scale,
70 - half2 (&z1z16)[2],
71 - half2 (&y1y16)[2]
72 -)
73 -{
74 - half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
75 - half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
55 +__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale(
56 + const uint32_t zero, const half scale, half2 (&z1z16)[2],
57 + half2 (&y1y16)[2]) {
58 + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
59 + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
60
77 - half2 scale2 = __half2half2(scale);
61 + half2 scale2 = __half2half2(scale);
62
79 - z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
80 - z1z16[1] = __hmul2(scale2, __half2half2(z16));
63 + z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
64 + z1z16[1] = __hmul2(scale2, __half2half2(z16));
65
82 - const half y1 = __float2half_rn(1.0f);
83 - const half y16 = __float2half_rn(1.0f / 16.0f);
66 + const half y1 = __float2half_rn(1.0f);
67 + const half y16 = __float2half_rn(1.0f / 16.0f);
68
85 - y1y16[0] = __hmul2(scale2, __half2half2(y1));
86 - y1y16[1] = __hmul2(scale2, __half2half2(y16));
69 + y1y16[0] = __hmul2(scale2, __half2half2(y1));
70 + y1y16[1] = __hmul2(scale2, __half2half2(y16));
71 }
72
89 -__forceinline__ __device__ void dequant_4bit_8_prep_zero
90 -(
91 - const uint32_t zero,
92 - half2(&z1z16)[2],
93 - half2(&y1y16)[2]
94 -)
95 -{
96 - half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
97 - half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
73 +__forceinline__ __device__ void dequant_4bit_8_prep_zero(const uint32_t zero,
74 + half2 (&z1z16)[2],
75 + half2 (&y1y16)[2]) {
76 + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
77 + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
78
99 - z1z16[0] = __half2half2(z1.as_half);
100 - z1z16[1] = __half2half2(z16);
79 + z1z16[0] = __half2half2(z1.as_half);
80 + z1z16[1] = __half2half2(z16);
81
102 - const half y1 = __float2half_rn(1.0f);
103 - const half y16 = __float2half_rn(1.0f / 16.0f);
82 + const half y1 = __float2half_rn(1.0f);
83 + const half y16 = __float2half_rn(1.0f / 16.0f);
84
105 - y1y16[0] = __half2half2(y1);
106 - y1y16[1] = __half2half2(y16);
85 + y1y16[0] = __half2half2(y1);
86 + y1y16[1] = __half2half2(y16);
87 }
88
109 -
110 -__forceinline__ __device__ void dequant_4bit_8_gptq
111 -(
112 - const uint32_t q_0,
113 - half2 (&dq)[4],
114 - half2 (&z1z16)[2],
115 - half2 (&y1y16)[2],
116 - int stride,
117 - bool scaled
118 -)
119 -{
120 - const uint32_t c0 = 0x64006400;
121 -
122 - uint32_t qa = q_0;
123 - half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )
124 - half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
125 - qa >>= 8;
126 - half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )
127 - half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
128 -
129 - if (scaled)
130 - {
131 - dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
132 - dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
133 - dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
134 - dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
135 - }
136 - else
137 - {
138 - dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
139 - dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )
140 - dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
141 - dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )
142 - }
89 +__forceinline__ __device__ void dequant_4bit_8_gptq(const uint32_t q_0,
90 + half2 (&dq)[4],
91 + half2 (&z1z16)[2],
92 + half2 (&y1y16)[2],
93 + int stride, bool scaled) {
94 + const uint32_t c0 = 0x64006400;
95 +
96 + uint32_t qa = q_0;
97 + half2_uint32 q0((qa & 0x000f000f) |
98 + c0); // half2( q[0] + 1024, q[1] + 1024 )
99 + half2_uint32 q1((qa & 0x00f000f0) |
100 + c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
101 + qa >>= 8;
102 + half2_uint32 q2((qa & 0x000f000f) |
103 + c0); // half2( q[4] + 1024, q[5] + 1024 )
104 + half2_uint32 q3((qa & 0x00f000f0) |
105 + c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
106 +
107 + if (scaled) {
108 + dq[0] = __hfma2(q0.as_half2, y1y16[0],
109 + z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
110 + dq[1] = __hfma2(q1.as_half2, y1y16[1],
111 + z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
112 + dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
113 + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
114 + } else {
115 + dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
116 + dq[1] = __hfma2(q1.as_half2, y1y16[1],
117 + z1z16[1]); // half2( q[2] - z, q[3] - z )
118 + dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
119 + dq[3] = __hfma2(q3.as_half2, y1y16[1],
120 + z1z16[1]); // half2( q[6] - z, q[7] - z )
121 + }
122 }
123 } // namespace gptq
124 } // namespace vllm
csrc/quantization/gptq/qdq_8.cuh
+12 -22
@@ -10,28 +10,18 @@ Copied from https://github.com/turboderp/exllamav2
10 namespace vllm {
11 namespace gptq {
12
13 -__forceinline__ __device__ void shuffle_8bit_4
14 -(
15 - uint32_t* q,
16 - int stride
17 -)
18 -{
19 -}
20 -
21 -__forceinline__ __device__ void dequant_8bit_8
22 -(
23 - const uint32_t q_0,
24 - const uint32_t q_1,
25 - half2 (&dq)[4],
26 - int stride,
27 - const uint32_t zero
28 -)
29 -{
30 - half dqh[8];
31 - for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), zero);
32 - for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
33 -
34 - for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
13 +__forceinline__ __device__ void shuffle_8bit_4(uint32_t* q, int stride) {}
14 +
15 +__forceinline__ __device__ void dequant_8bit_8(const uint32_t q_0,
16 + const uint32_t q_1,
17 + half2 (&dq)[4], int stride,
18 + const uint32_t zero) {
19 + half dqh[8];
20 + for (int i = 0; i < 4; i++) dqh[i] = dq_ns(exb(q_0, i * 8, 0xff), zero);
21 + for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
22 +
23 + for (int i = 0; i < 4; i++)
24 + dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
25 }
26
27 } // namespace gptq
csrc/quantization/gptq/qdq_util.cuh
+27 -31
@@ -8,51 +8,47 @@ Copied from https://github.com/turboderp/exllamav2
8 namespace vllm {
9 namespace gptq {
10
11 -union half2_uint32
12 -{
13 - uint32_t as_uint32;
14 - half2 as_half2;
15 - __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
16 - __device__ half2_uint32(half2 val) : as_half2(val) {}
11 +union half2_uint32 {
12 + uint32_t as_uint32;
13 + half2 as_half2;
14 + __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
15 + __device__ half2_uint32(half2 val) : as_half2(val) {}
16 };
17
19 -union half_uint16
20 -{
21 - uint16_t as_uint16;
22 - half as_half;
23 - __device__ half_uint16(uint16_t val) : as_uint16(val) {}
24 - __device__ half_uint16(half val) : as_half(val) {}
18 +union half_uint16 {
19 + uint16_t as_uint16;
20 + half as_half;
21 + __device__ half_uint16(uint16_t val) : as_uint16(val) {}
22 + __device__ half_uint16(half val) : as_half(val) {}
23 };
24
25 // Max_scale premultiplied by 1/256
26
29 -__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)
30 -{
31 - int qs_i = qs + 1;
32 - half qs_h = __int2half_rn(qs_i * qs_i);
33 - qs_h = __hmul(qs_h, max_scale);
34 - return qs_h;
27 +__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) {
28 + int qs_i = qs + 1;
29 + half qs_h = __int2half_rn(qs_i * qs_i);
30 + qs_h = __hmul(qs_h, max_scale);
31 + return qs_h;
32 }
33
37 -__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)
38 -{
39 - return __hmul(__int2half_rn(q - qzero), scale);
34 +__forceinline__ __device__ half dq(const int q, const int qzero,
35 + const half scale) {
36 + return __hmul(__int2half_rn(q - qzero), scale);
37 }
38
42 -__forceinline__ __device__ half dq_ns(const int q, const int qzero)
43 -{
44 - //return __hsub(__int2half_rn(q), __int2half_rn(qzero));
45 - return __int2half_rn(q - qzero);
39 +__forceinline__ __device__ half dq_ns(const int q, const int qzero) {
40 + // return __hsub(__int2half_rn(q), __int2half_rn(qzero));
41 + return __int2half_rn(q - qzero);
42 }
43
48 -__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)
49 -{
50 - return (int)((q >> shift) & mask);
44 +__forceinline__ __device__ int exb(const uint32_t q, const int shift,
45 + const int mask) {
46 + return (int)((q >> shift) & mask);
47 }
48
53 -__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)
54 -{
55 - return (int)(__funnelshift_rc(q0, q1, shift) & mask);
49 +__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0,
50 + const int shift, const int mask) {
51 + return (int)(__funnelshift_rc(q0, q1, shift) & mask);
52 }
53
54 } // namespace gptq
csrc/quantization/gptq_marlin/gptq_marlin.cu
+360 -336
@@ -22,53 +22,58 @@
22 #include "gptq_marlin.cuh"
23 #include "gptq_marlin_dtypes.cuh"
24
25 -#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) static_assert(\
26 - std::is_same<scalar_t, half>::value || std::is_same<scalar_t, nv_bfloat16>::value, \
27 - "only float16 and bfloat16 is supported");
28 -
29 -template <typename T> inline std::string str(T x) { return std::to_string(x); }
25 +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
26 + static_assert(std::is_same<scalar_t, half>::value || \
27 + std::is_same<scalar_t, nv_bfloat16>::value, \
28 + "only float16 and bfloat16 is supported");
29 +
30 +template <typename T>
31 +inline std::string str(T x) {
32 + return std::to_string(x);
33 +}
34
35 namespace gptq_marlin {
36
37 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38
35 -__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
36 - int const *__restrict__ perm_int_ptr,
37 - int4 *__restrict__ out_int4_ptr, int size_m,
39 +__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
40 + int const* __restrict__ perm_int_ptr,
41 + int4* __restrict__ out_int4_ptr, int size_m,
42 int size_k, int block_rows) {}
43
40 -template <typename scalar_t, // compute dtype, half or nv_float16
41 - const int num_bits, // number of bits used for weights
42 - const int threads, // number of threads in a threadblock
43 - const int thread_m_blocks, // number of 16x16 blocks in the m
44 - // dimension (batchsize) of the threadblock
45 - const int thread_n_blocks, // same for n dimension (output)
46 - const int thread_k_blocks, // same for k dimension (reduction)
47 - const int stages, // number of stages for the async global->shared
48 - // fetch pipeline
49 - const bool has_act_order, // whether act_order is enabled
50 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
51 - // a separate quantization scale
44 +template <typename scalar_t, // compute dtype, half or nv_float16
45 + const int num_bits, // number of bits used for weights
46 + const int threads, // number of threads in a threadblock
47 + const int thread_m_blocks, // number of 16x16 blocks in the m
48 + // dimension (batchsize) of the
49 + // threadblock
50 + const int thread_n_blocks, // same for n dimension (output)
51 + const int thread_k_blocks, // same for k dimension (reduction)
52 + const int stages, // number of stages for the async global->shared
53 + // fetch pipeline
54 + const bool has_act_order, // whether act_order is enabled
55 + const int group_blocks = -1 // number of consecutive 16x16 blocks
56 + // with a separate quantization scale
57 >
53 -__global__ void
54 -Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
55 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
56 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
57 - const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
58 - // (k/groupsize)xn
59 - const int *__restrict__ g_idx, // int32 group indices of shape k
60 - int num_groups, // number of scale groups per output channel
61 - int prob_m, // batch dimension m
62 - int prob_n, // output dimension n
63 - int prob_k, // reduction dimension k
64 - int *locks // extra global storage for barrier synchronization
58 +__global__ void Marlin(
59 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
60 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
61 + int4* __restrict__ C, // fp16 output buffer of shape mxn
62 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
63 + // (k/groupsize)xn
64 + const int* __restrict__ g_idx, // int32 group indices of shape k
65 + int num_groups, // number of scale groups per output channel
66 + int prob_m, // batch dimension m
67 + int prob_n, // output dimension n
68 + int prob_k, // reduction dimension k
69 + int* locks // extra global storage for barrier synchronization
70 ) {}
71
67 -} // namespace gptq_marlin
72 +} // namespace gptq_marlin
73
69 -torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
70 - torch::Tensor &b_scales, torch::Tensor &g_idx,
71 - torch::Tensor &perm, torch::Tensor &workspace,
74 +torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
75 + torch::Tensor& b_scales, torch::Tensor& g_idx,
76 + torch::Tensor& perm, torch::Tensor& workspace,
77 int64_t num_bits, int64_t size_m, int64_t size_n,
78 int64_t size_k, bool is_k_full) {
79 TORCH_CHECK_NOT_IMPLEMENTED(false,
@@ -81,24 +86,26 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
86 // m16n8k16 tensor core mma instruction with fp16 inputs and fp32
87 // output/accumulation.
88 template <typename scalar_t>
84 -__device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
85 - const typename ScalarType<scalar_t>::FragB &frag_b,
86 - typename ScalarType<scalar_t>::FragC &frag_c) {
87 - const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
88 - const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
89 - float *c = reinterpret_cast<float *>(&frag_c);
89 +__device__ inline void mma(const typename ScalarType<scalar_t>::FragA& a_frag,
90 + const typename ScalarType<scalar_t>::FragB& frag_b,
91 + typename ScalarType<scalar_t>::FragC& frag_c) {
92 + const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
93 + const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
94 + float* c = reinterpret_cast<float*>(&frag_c);
95 if constexpr (std::is_same<scalar_t, half>::value) {
91 - asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
92 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
93 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
94 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
95 - "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
96 + asm volatile(
97 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
98 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
101 + "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
102 } else if constexpr (std::is_same<scalar_t, nv_bfloat16>::value) {
97 - asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
98 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
101 - "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
103 + asm volatile(
104 + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
105 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
106 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
107 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
108 + "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
109 } else {
110 STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
111 }
@@ -107,8 +114,9 @@ __device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
114 // Instruction for loading a full 16x16 matrix fragment of operand A from shared
115 // memory, directly in tensor core layout.
116 template <typename scalar_t>
110 -__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const void *smem_ptr) {
111 - uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
117 +__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA& frag_a,
118 + const void* smem_ptr) {
119 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
120 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
121 asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
122 : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
@@ -118,7 +126,8 @@ __device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const
126 // Lookup-table based 3-input logical operation; explicitly used for
127 // dequantization as the compiler does not seem to automatically recognize it in
128 // all cases.
121 -template <int lut> __device__ inline int lop3(int a, int b, int c) {
129 +template <int lut>
130 +__device__ inline int lop3(int a, int b, int c) {
131 int res;
132 asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
133 : "=r"(res)
@@ -140,8 +149,10 @@ __device__ inline uint32_t prmt(uint32_t a) {
149 // Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
150 // values. We mostly follow the strategy in the link below, with some small
151 // changes:
143 -// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
144 -// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
152 +// - FP16:
153 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154 +// - BF16:
155 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
156 template <typename scalar_t>
157 __device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
158 STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
@@ -161,16 +172,17 @@ __device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
172 const int MUL = 0x2c002c00;
173 const int ADD = 0xd480d480;
174 typename ScalarType<half>::FragB frag_b;
164 - frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165 - *reinterpret_cast<const half2 *>(&SUB));
166 - frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
167 - *reinterpret_cast<const half2 *>(&MUL),
168 - *reinterpret_cast<const half2 *>(&ADD));
175 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
176 + *reinterpret_cast<const half2*>(&SUB));
177 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
178 + *reinterpret_cast<const half2*>(&MUL),
179 + *reinterpret_cast<const half2*>(&ADD));
180 return frag_b;
181 }
182
183 template <>
173 -__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat16>(int q) {
184 +__device__ inline typename ScalarType<nv_bfloat16>::FragB
185 +dequant_4bit<nv_bfloat16>(int q) {
186 static constexpr uint32_t MASK = 0x000f000f;
187 static constexpr uint32_t EX = 0x43004300;
188
@@ -184,7 +196,7 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat
196 static constexpr uint32_t MUL = 0x3F803F80;
197 static constexpr uint32_t ADD = 0xC308C308;
198
187 - frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162 *>(&lo),
199 + frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
200 *reinterpret_cast<const nv_bfloat162*>(&MUL),
201 *reinterpret_cast<const nv_bfloat162*>(&ADD));
202 frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
@@ -193,10 +205,12 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat
205 return frag_b;
206 }
207
196 -// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or bf16
197 -// Reference:
198 -// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
199 -// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
208 +// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or
209 +// bf16 Reference:
210 +// - FP16:
211 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
212 +// - BF16:
213 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
214 template <typename scalar_t>
215 __device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
216 STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
@@ -214,24 +228,26 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
228 static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
229
230 typename ScalarType<half>::FragB frag_b;
217 - frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
218 - *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
219 - frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
220 - *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
231 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
232 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
233 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
234 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
235 return frag_b;
236 }
237
238 template <>
225 -__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat16>(int q) {
239 +__device__ inline typename ScalarType<nv_bfloat16>::FragB
240 +dequant_8bit<nv_bfloat16>(int q) {
241 typename ScalarType<nv_bfloat16>::FragB frag_b;
242
243 float fp32_intermediates[4];
229 - uint32_t * fp32_intermediates_casted = reinterpret_cast<uint32_t*>(fp32_intermediates);
244 + uint32_t* fp32_intermediates_casted =
245 + reinterpret_cast<uint32_t*>(fp32_intermediates);
246
247 static constexpr uint32_t fp32_base = 0x4B000000;
232 - fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
248 + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
249 fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
234 - fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
250 + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
251 fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
252
253 fp32_intermediates[0] -= 8388736.f;
@@ -240,8 +256,10 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat
256 fp32_intermediates[3] -= 8388736.f;
257
258 uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
243 - bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632);
244 - bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632);
259 + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0],
260 + fp32_intermediates_casted[1], 0x7632);
261 + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2],
262 + fp32_intermediates_casted[3], 0x7632);
263
264 return frag_b;
265 }
@@ -249,30 +267,32 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat
267 // Multiply dequantized values by the corresponding quantization scale; used
268 // only for grouped quantization.
269 template <typename scalar_t>
252 -__device__ inline void scale(typename ScalarType<scalar_t>::FragB &frag_b,
253 - typename ScalarType<scalar_t>::FragS &frag_s, int i) {
270 +__device__ inline void scale(typename ScalarType<scalar_t>::FragB& frag_b,
271 + typename ScalarType<scalar_t>::FragS& frag_s,
272 + int i) {
273 using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
255 - scalar_t2 s = ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t *>(&frag_s)[i]);
274 + scalar_t2 s =
275 + ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t*>(&frag_s)[i]);
276 frag_b[0] = __hmul2(frag_b[0], s);
277 frag_b[1] = __hmul2(frag_b[1], s);
278 }
279
280 // Same as above, but for act_order (each K is multiplied individually)
281 template <typename scalar_t>
262 -__device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
263 - typename ScalarType<scalar_t>::FragS &frag_s_1,
264 - typename ScalarType<scalar_t>::FragS &frag_s_2,
265 - typename ScalarType<scalar_t>::FragS &frag_s_3,
266 - typename ScalarType<scalar_t>::FragS &frag_s_4,
282 +__device__ inline void scale4(typename ScalarType<scalar_t>::FragB& frag_b,
283 + typename ScalarType<scalar_t>::FragS& frag_s_1,
284 + typename ScalarType<scalar_t>::FragS& frag_s_2,
285 + typename ScalarType<scalar_t>::FragS& frag_s_3,
286 + typename ScalarType<scalar_t>::FragS& frag_s_4,
287 int i) {
268 - using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
288 + using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
289 scalar_t2 s_val_1_2;
270 - s_val_1_2.x = reinterpret_cast<scalar_t *>(&frag_s_1)[i];
271 - s_val_1_2.y = reinterpret_cast<scalar_t *>(&frag_s_2)[i];
290 + s_val_1_2.x = reinterpret_cast<scalar_t*>(&frag_s_1)[i];
291 + s_val_1_2.y = reinterpret_cast<scalar_t*>(&frag_s_2)[i];
292
293 scalar_t2 s_val_3_4;
274 - s_val_3_4.x = reinterpret_cast<scalar_t *>(&frag_s_3)[i];
275 - s_val_3_4.y = reinterpret_cast<scalar_t *>(&frag_s_4)[i];
294 + s_val_3_4.x = reinterpret_cast<scalar_t*>(&frag_s_3)[i];
295 + s_val_3_4.y = reinterpret_cast<scalar_t*>(&frag_s_4)[i];
296
297 frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
298 frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
@@ -280,14 +300,15 @@ __device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
300
301 // Given 2 floats multiply by 2 scales (halves)
302 template <typename scalar_t>
283 -__device__ inline void scale_float(float *c, typename ScalarType<scalar_t>::FragS &s) {
284 - scalar_t *s_ptr = reinterpret_cast<scalar_t *>(&s);
303 +__device__ inline void scale_float(float* c,
304 + typename ScalarType<scalar_t>::FragS& s) {
305 + scalar_t* s_ptr = reinterpret_cast<scalar_t*>(&s);
306 c[0] = __fmul_rn(c[0], ScalarType<scalar_t>::num2float(s_ptr[0]));
307 c[1] = __fmul_rn(c[1], ScalarType<scalar_t>::num2float(s_ptr[1]));
308 }
309
310 // Wait until barrier reaches `count`, then lock for current threadblock.
290 -__device__ inline void barrier_acquire(int *lock, int count) {
311 +__device__ inline void barrier_acquire(int* lock, int count) {
312 if (threadIdx.x == 0) {
313 int state = -1;
314 do
@@ -302,7 +323,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
323 }
324
325 // Release barrier and increment visitation count.
305 -__device__ inline void barrier_release(int *lock, bool reset = false) {
326 +__device__ inline void barrier_release(int* lock, bool reset = false) {
327 __syncthreads();
328 if (threadIdx.x == 0) {
329 if (reset) {
@@ -321,11 +342,10 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
342
343 // For a given "a" of size [M,K] performs a permutation of the K columns based
344 // on the given "perm" indices.
324 -__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
325 - int const *__restrict__ perm_int_ptr,
326 - int4 *__restrict__ out_int4_ptr, int size_m,
345 +__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
346 + int const* __restrict__ perm_int_ptr,
347 + int4* __restrict__ out_int4_ptr, int size_m,
348 int size_k, int block_rows) {
328 -
349 int start_row = block_rows * blockIdx.x;
350 int finish_row = start_row + block_rows;
351 if (finish_row > size_m) {
@@ -341,9 +361,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
361
362 int offset = row * row_stride;
363
344 - half const *a_row_half =
345 - reinterpret_cast<half const *>(a_int4_ptr + offset);
346 - half *out_half = reinterpret_cast<half *>(out_int4_ptr + offset);
364 + half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
365 + half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
366
367 int base_k = 0;
368
@@ -374,31 +393,32 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
393 }
394 }
395
377 -template <typename scalar_t, // compute dtype, half or nv_float16
378 - const int num_bits, // number of bits used for weights
379 - const int threads, // number of threads in a threadblock
380 - const int thread_m_blocks, // number of 16x16 blocks in the m
381 - // dimension (batchsize) of the threadblock
382 - const int thread_n_blocks, // same for n dimension (output)
383 - const int thread_k_blocks, // same for k dimension (reduction)
384 - const int stages, // number of stages for the async global->shared
385 - // fetch pipeline
386 - const bool has_act_order, // whether act_order is enabled
387 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
388 - // a separate quantization scale
396 +template <typename scalar_t, // compute dtype, half or nv_float16
397 + const int num_bits, // number of bits used for weights
398 + const int threads, // number of threads in a threadblock
399 + const int thread_m_blocks, // number of 16x16 blocks in the m
400 + // dimension (batchsize) of the
401 + // threadblock
402 + const int thread_n_blocks, // same for n dimension (output)
403 + const int thread_k_blocks, // same for k dimension (reduction)
404 + const int stages, // number of stages for the async global->shared
405 + // fetch pipeline
406 + const bool has_act_order, // whether act_order is enabled
407 + const int group_blocks = -1 // number of consecutive 16x16 blocks
408 + // with a separate quantization scale
409 >
390 -__global__ void
391 -Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
392 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
393 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
394 - const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
395 - // (k/groupsize)xn
396 - const int *__restrict__ g_idx, // int32 group indices of shape k
397 - int num_groups, // number of scale groups per output channel
398 - int prob_m, // batch dimension m
399 - int prob_n, // output dimension n
400 - int prob_k, // reduction dimension k
401 - int *locks // extra global storage for barrier synchronization
410 +__global__ void Marlin(
411 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
412 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
413 + int4* __restrict__ C, // fp16 output buffer of shape mxn
414 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
415 + // (k/groupsize)xn
416 + const int* __restrict__ g_idx, // int32 group indices of shape k
417 + int num_groups, // number of scale groups per output channel
418 + int prob_m, // batch dimension m
419 + int prob_n, // output dimension n
420 + int prob_k, // reduction dimension k
421 + int* locks // extra global storage for barrier synchronization
422 ) {
423 // Each threadblock processes one "stripe" of the B matrix with (roughly) the
424 // same size, which might involve multiple column "slices" (of width 16 *
@@ -445,11 +465,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
465 int slice_row = (iters * blockIdx.x) % k_tiles;
466 int slice_col_par = (iters * blockIdx.x) / k_tiles;
467 int slice_col = slice_col_par;
448 - int slice_iters; // number of threadblock tiles in the current slice
468 + int slice_iters; // number of threadblock tiles in the current slice
469 int slice_count =
450 - 0; // total number of active threadblocks in the current slice
451 - int slice_idx; // index of threadblock in current slice; numbered bottom to
452 - // top
470 + 0; // total number of active threadblocks in the current slice
471 + int slice_idx; // index of threadblock in current slice; numbered bottom to
472 + // top
473
474 // We can easily implement parallel problem execution by just remapping
475 // indices and advancing global pointers
@@ -465,27 +485,22 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
485 auto init_slice = [&]() {
486 slice_iters =
487 iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
468 - if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
469 - slice_iters = 0;
470 - if (slice_iters == 0)
471 - return;
472 - if (slice_row + slice_iters > k_tiles)
473 - slice_iters = k_tiles - slice_row;
488 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
489 + if (slice_iters == 0) return;
490 + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
491 slice_count = 1;
492 slice_idx = 0;
493 int col_first = iters * div_ceil(k_tiles * slice_col_par, iters);
494 if (col_first <= k_tiles * (slice_col_par + 1)) {
495 int col_off = col_first - k_tiles * slice_col_par;
496 slice_count = div_ceil(k_tiles - col_off, iters);
480 - if (col_off > 0)
481 - slice_count++;
497 + if (col_off > 0) slice_count++;
498 int delta_first = iters * blockIdx.x - col_first;
499 if (delta_first < 0 || (col_off == 0 && delta_first == 0))
500 slice_idx = slice_count - 1;
501 else {
502 slice_idx = slice_count - 1 - delta_first / iters;
487 - if (col_off > 0)
488 - slice_idx--;
503 + if (col_off > 0) slice_idx--;
504 }
505 }
506 if (slice_col == n_tiles) {
@@ -605,7 +620,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
620 // needed if there are more threads than required for a certain tilesize or
621 // when the batchsize is not a multiple of 16.
622 bool a_sh_wr_pred[a_sh_wr_iters];
608 -#pragma unroll
623 + #pragma unroll
624 for (int i = 0; i < a_sh_wr_iters; i++)
625 a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
626
@@ -623,13 +638,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
638 // loop unrolls, all shared memory accesses are static, we simply precompute
639 // both transformed reads and writes.
640 int a_sh_wr_trans[a_sh_wr_iters];
626 -#pragma unroll
641 + #pragma unroll
642 for (int i = 0; i < a_sh_wr_iters; i++)
643 a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
644 int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
630 -#pragma unroll
645 + #pragma unroll
646 for (int i = 0; i < b_sh_wr_iters; i++) {
632 -#pragma unroll
647 + #pragma unroll
648 for (int j = 0; j < thread_m_blocks; j++)
649 a_sh_rd_trans[i][j] =
650 transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -639,30 +654,30 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
654 // runtime; we break dependencies between subsequent accesses with a tile by
655 // maintining multiple pointers (we have enough registers), a tiny
656 // optimization.
642 - const int4 *B_ptr[b_sh_wr_iters];
643 -#pragma unroll
657 + const int4* B_ptr[b_sh_wr_iters];
658 + #pragma unroll
659 for (int i = 0; i < b_sh_wr_iters; i++)
660 B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
661
662 extern __shared__ int4 sh[];
663 // Shared memory storage for global fetch pipelines.
649 - int4 *sh_a = sh;
650 - int4 *sh_b = sh_a + (stages * a_sh_stage);
651 - int4 *sh_g_idx = sh_b + (stages * b_sh_stage);
652 - int4 *sh_s = sh_g_idx + (stages * g_idx_stage);
664 + int4* sh_a = sh;
665 + int4* sh_b = sh_a + (stages * a_sh_stage);
666 + int4* sh_g_idx = sh_b + (stages * b_sh_stage);
667 + int4* sh_s = sh_g_idx + (stages * g_idx_stage);
668
669 // Register storage for double buffer of shared memory reads.
670 FragA frag_a[2][thread_m_blocks];
671 I4 frag_b_quant[2][b_thread_vecs];
672 FragC frag_c[thread_m_blocks][4][2];
658 - FragS frag_s[2][4]; // No act-order
659 - FragS act_frag_s[2][4][4]; // For act-order
673 + FragS frag_s[2][4]; // No act-order
674 + FragS act_frag_s[2][4][4]; // For act-order
675
676 // Zero accumulators.
677 auto zero_accums = [&]() {
663 -#pragma unroll
678 + #pragma unroll
679 for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
665 - reinterpret_cast<float *>(frag_c)[i] = 0;
680 + reinterpret_cast<float*>(frag_c)[i] = 0;
681 };
682
683 int sh_first_group_id = -1;
@@ -706,18 +721,18 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
721 // shared memory pipeline location.
722 auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
723 if (pred) {
709 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
710 -#pragma unroll
724 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
725 + #pragma unroll
726 for (int i = 0; i < a_sh_wr_iters; i++) {
727 cp_async4_pred(
728 &sh_a_stage[a_sh_wr_trans[i]],
729 &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
730 a_sh_wr_pred[i]);
731 }
717 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
718 -#pragma unroll
732 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
733 + #pragma unroll
734 for (int i = 0; i < b_sh_wr_iters; i++) {
720 -#pragma unroll
735 + #pragma unroll
736 for (int j = 0; j < b_thread_vecs; j++) {
737 cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
738 }
@@ -730,10 +745,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
745 int full_pipe = a_off;
746 int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
747 if (cur_k < prob_k && cur_k < slice_k_finish) {
733 - int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
748 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
749
735 - int4 const *cur_g_idx_stage_ptr =
736 - reinterpret_cast<int4 const *>(&g_idx[cur_k]);
750 + int4 const* cur_g_idx_stage_ptr =
751 + reinterpret_cast<int4 const*>(&g_idx[cur_k]);
752
753 if (threadIdx.x < g_idx_stage) {
754 cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
@@ -742,7 +757,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
757 }
758 } else {
759 if constexpr (group_blocks != -1) {
745 - int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
760 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
761
762 if constexpr (group_blocks >= thread_k_blocks) {
763 // Only fetch scales if this tile starts a new group
@@ -782,15 +797,16 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
797 // Load the next sub-tile from the current location in the shared memory pipe
798 // into the current register buffer.
799 auto fetch_to_registers = [&](int k, int pipe) {
785 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
786 -#pragma unroll
800 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
801 + #pragma unroll
802 for (int i = 0; i < thread_m_blocks; i++)
788 - ldsm4<scalar_t>(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
789 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
803 + ldsm4<scalar_t>(frag_a[k % 2][i],
804 + &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
805 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
806
791 -#pragma unroll
807 + #pragma unroll
808 for (int i = 0; i < b_thread_vecs; i++) {
793 - frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
809 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
810 &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
811 }
812 };
@@ -805,8 +821,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
821 return;
822 }
823
808 - int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
809 - int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
824 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
825 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
826
827 int group_id_1 = sh_g_idx_int_ptr[0];
828 int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
@@ -822,10 +838,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
838 // No act-order case
839 if constexpr (group_blocks != -1) {
840 if constexpr (group_blocks >= thread_k_blocks) {
825 - int4 *sh_s_stage =
841 + int4* sh_s_stage =
842 sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
843 (pipe / (group_blocks / thread_k_blocks)));
828 - reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
844 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
845 } else {
846 int warp_id = threadIdx.x / 32;
847 int n_warps = thread_n_blocks / 4;
@@ -838,9 +854,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
854 int k_blocks = cur_k / 16;
855 int cur_group_id = k_blocks / group_blocks;
856
841 - int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
857 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
858
843 - reinterpret_cast<int4 *>(&frag_s[k % 2])[0] =
859 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
860 sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
861 }
862 }
@@ -867,7 +883,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
883 // thread-id)
884 int warp_id = threadIdx.x / 32;
885 int n_warps =
870 - thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
886 + thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
887
888 int warp_row = warp_id / n_warps;
889 int warp_col = warp_id % n_warps;
@@ -875,7 +891,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
891 cur_k += warp_row * 16;
892
893 int th_id = threadIdx.x % 32;
878 - cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
894 + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
895
896 int s_col_shift =
897 /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
@@ -883,45 +899,44 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
899
900 if (is_same_group[pipe]) {
901 if (k % 2 == 0) {
886 - *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
902 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
903 sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
904 s_col_shift];
905 } else {
890 - *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
891 - *(reinterpret_cast<int4 *>(&(act_frag_s[(k - 1) % 2][0][0])));
906 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
907 + *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
908 }
909
910 for (int i = 1; i < 4; i++) {
895 - *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
896 - *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0])));
911 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
912 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
913 }
914 return;
915 }
916
901 - int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
902 - int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
917 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
918 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
919
920 constexpr int k_frag_offsets[4] = {0, 1, 8,
905 - 9}; // Tensor core offsets per thread
921 + 9}; // Tensor core offsets per thread
922
907 -#pragma unroll
923 + #pragma unroll
924 for (int i = 0; i < 4; i++) {
909 -
925 int actual_k = cur_k + k_frag_offsets[i];
926
927 int group_id = sh_g_idx_int_ptr[actual_k];
928 int rel_group_id = group_id - sh_first_group_id;
929
915 - *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
930 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
931 sh_s[rel_group_id * s_sh_stride + s_col_shift];
932 }
933 };
934
935 // Execute the actual tensor core matmul of a sub-tile.
936 auto matmul = [&](int k) {
922 -// We have the m dimension as the inner loop in order to encourage overlapping
923 -// dequantization and matmul operations.
924 -#pragma unroll
937 + // We have the m dimension as the inner loop in order to encourage overlapping
938 + // dequantization and matmul operations.
939 + #pragma unroll
940 for (int j = 0; j < 4; j++) {
941 FragB frag_b0;
942 FragB frag_b1;
@@ -933,7 +948,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
948 frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
949
950 } else {
936 - int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
951 + int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
952 int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
953 int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
954
@@ -943,8 +958,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
958
959 // Apply scale to frag_b0
960 if constexpr (has_act_order) {
946 - scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
947 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
961 + scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j],
962 + act_frag_s[k % 2][1][j], act_frag_s[k % 2][2][j],
963 + act_frag_s[k % 2][3][j], 0);
964 } else {
965 if constexpr (group_blocks != -1) {
966 scale<scalar_t>(frag_b0, frag_s[k % 2][j], 0);
@@ -953,8 +969,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
969
970 // Apply scale to frag_b1
971 if constexpr (has_act_order) {
956 - scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
957 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
972 + scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
973 + act_frag_s[k % 2][1][j], act_frag_s[k % 2][2][j],
974 + act_frag_s[k % 2][3][j], 1);
975
976 } else {
977 if constexpr (group_blocks != -1) {
@@ -962,7 +979,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
979 }
980 }
981
965 -#pragma unroll
982 + #pragma unroll
983 for (int i = 0; i < thread_m_blocks; i++) {
984 mma<scalar_t>(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
985 mma<scalar_t>(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
@@ -987,38 +1004,38 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1004 // unnecessary read or write iterations, e.g., for two warps we write only
1005 // once by warp 1 and read only once by warp 0.
1006
990 -#pragma unroll
1007 + #pragma unroll
1008 for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
992 -#pragma unroll
1009 + #pragma unroll
1010 for (int i = red_off; i > 0; i /= 2) {
1011 if (i <= red_idx && red_idx < 2 * i) {
995 -#pragma unroll
1012 + #pragma unroll
1013 for (int j = 0; j < 4 * 2; j++) {
1014 int red_sh_wr =
1015 red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
1016 if (i < red_off) {
1000 - float *c_rd = reinterpret_cast<float *>(
1001 - &sh[red_sh_delta * j + red_sh_rd]);
1002 - float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
1003 -#pragma unroll
1017 + float* c_rd =
1018 + reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
1019 + float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
1020 + #pragma unroll
1021 for (int k = 0; k < 4; k++)
1005 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
1022 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
1023 c_rd[k] + c_wr[k];
1024 }
1025 sh[red_sh_wr] =
1009 - reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
1026 + reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
1027 }
1028 }
1029 __syncthreads();
1030 }
1031 if (red_idx == 0) {
1015 -#pragma unroll
1032 + #pragma unroll
1033 for (int i = 0; i < 4 * 2; i++) {
1017 - float *c_rd =
1018 - reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
1019 -#pragma unroll
1034 + float* c_rd =
1035 + reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
1036 + #pragma unroll
1037 for (int j = 0; j < 4; j++)
1021 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
1038 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
1039 c_rd[j];
1040 }
1041 }
@@ -1049,39 +1066,39 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1066 int row = (threadIdx.x % 32) / 4;
1067
1068 if (!first) {
1052 -// Interestingly, doing direct global accesses here really seems to mess up the
1053 -// compiler and lead to slowdowns, hence we also use async-copies even though
1054 -// these fetches are not actually asynchronous.
1055 -#pragma unroll
1069 + // Interestingly, doing direct global accesses here really seems to mess up
1070 + // the compiler and lead to slowdowns, hence we also use async-copies even
1071 + // though these fetches are not actually asynchronous.
1072 + #pragma unroll
1073 for (int i = 0; i < thread_m_blocks * 4; i++) {
1057 - cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
1058 - &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
1059 - c_gl_wr_delta_i * (i % 2)],
1060 - i < (thread_m_blocks - 1) * 4 ||
1061 - 8 * (i / 2) + row < prob_m);
1074 + cp_async4_pred(
1075 + &sh[c_sh_wr + c_sh_wr_delta * i],
1076 + &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
1077 + c_gl_wr_delta_i * (i % 2)],
1078 + i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
1079 }
1080 cp_async_fence();
1081 cp_async_wait<0>();
1082 }
1083
1067 -#pragma unroll
1084 + #pragma unroll
1085 for (int i = 0; i < thread_m_blocks * 4; i++) {
1086 if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
1087 if (!first) {
1088 int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
1072 -#pragma unroll
1089 + #pragma unroll
1090 for (int j = 0; j < 2 * 4; j++) {
1074 - reinterpret_cast<float *>(
1091 + reinterpret_cast<float*>(
1092 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
1076 - Dtype::num2float(reinterpret_cast<scalar_t *>(&c_red)[j]);
1093 + Dtype::num2float(reinterpret_cast<scalar_t*>(&c_red)[j]);
1094 }
1095 }
1096 if (!last) {
1097 int4 c;
1081 -#pragma unroll
1098 + #pragma unroll
1099 for (int j = 0; j < 2 * 4; j++) {
1083 - reinterpret_cast<scalar_t *>(&c)[j] =
1084 - Dtype::float2num(reinterpret_cast<float *>(
1100 + reinterpret_cast<scalar_t*>(&c)[j] =
1101 + Dtype::float2num(reinterpret_cast<float*>(
1102 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1103 }
1104 C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -1115,8 +1132,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1132
1133 // We first reorder in shared memory to guarantee the most efficient final
1134 // global write patterns
1118 - auto write = [&](int idx, float c0, float c1, FragS &s) {
1119 - scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1135 + auto write = [&](int idx, float c0, float c1, FragS& s) {
1136 + scalar_t2 res =
1137 + Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1138
1139 // For per-column quantization we finally apply the scale here (only for
1140 // 4-bit)
@@ -1124,13 +1142,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1142 res = __hmul2(res, s[0]);
1143 }
1144
1127 - ((scalar_t2 *)sh)[idx] = res;
1145 + ((scalar_t2*)sh)[idx] = res;
1146 };
1147
1148 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1131 -#pragma unroll
1149 + #pragma unroll
1150 for (int i = 0; i < thread_m_blocks; i++) {
1133 -#pragma unroll
1151 + #pragma unroll
1152 for (int j = 0; j < 4; j++) {
1153 int wr = c_sh_wr + 8 * j;
1154 write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
@@ -1147,7 +1165,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1165 }
1166 __syncthreads();
1167
1150 -#pragma unroll
1168 + #pragma unroll
1169 for (int i = 0;
1170 i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1171 i++) {
@@ -1162,7 +1180,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1180 // Start global fetch and register load pipelines.
1181 auto start_pipes = [&]() {
1182
1165 -#pragma unroll
1183 + #pragma unroll
1184 for (int i = 0; i < stages - 1; i++) {
1185 if (has_act_order && i == 0) {
1186 int last_g_idx = slice_k_start + stages * tb_k * 2;
@@ -1193,9 +1211,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1211 // have even length meaning that the next iteration will always start at
1212 // index 0.
1213
1196 -#pragma unroll
1214 + #pragma unroll
1215 for (int pipe = 0; pipe < stages;) {
1198 -#pragma unroll
1216 + #pragma unroll
1217 for (int k = 0; k < b_sh_wr_iters; k++) {
1218 fetch_to_registers(k + 1, pipe % stages);
1219 fetch_scales_to_registers(k + 1, pipe);
@@ -1261,8 +1279,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1279 cp_async_wait<0>();
1280 __syncthreads();
1281 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264 - reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265 - reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1282 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1283 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1284 }
1285
1286 } else {
@@ -1270,8 +1288,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1288 cp_async_wait<0>();
1289 __syncthreads();
1290 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273 - reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274 - reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1291 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1292 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1293 }
1294 }
1295 }
@@ -1282,31 +1300,35 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1300 // overflow in fp16)
1301 if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1302 if (threadIdx.x / 32 < thread_n_blocks / 4) {
1285 -#pragma unroll
1303 + #pragma unroll
1304 for (int i = 0; i < thread_m_blocks; i++) {
1287 -#pragma unroll
1305 + #pragma unroll
1306 for (int j = 0; j < 4; j++) {
1289 - scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1290 - frag_s[j / 2][2 * (j % 2) + 0]);
1291 - scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1292 - frag_s[j / 2][2 * (j % 2) + 0]);
1293 -
1294 - scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1295 - frag_s[j / 2][2 * (j % 2) + 1]);
1296 - scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1297 - frag_s[j / 2][2 * (j % 2) + 1]);
1307 + scale_float<scalar_t>(
1308 + reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1309 + frag_s[j / 2][2 * (j % 2) + 0]);
1310 + scale_float<scalar_t>(
1311 + reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1312 + frag_s[j / 2][2 * (j % 2) + 0]);
1313 +
1314 + scale_float<scalar_t>(
1315 + reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1316 + frag_s[j / 2][2 * (j % 2) + 1]);
1317 + scale_float<scalar_t>(
1318 + reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1319 + frag_s[j / 2][2 * (j % 2) + 1]);
1320 }
1321 }
1322 }
1323 }
1324
1303 - if (slice_count > 1) { // only globally reduce if there is more than one
1304 - // block in a slice
1325 + if (slice_count > 1) { // only globally reduce if there is more than one
1326 + // block in a slice
1327 barrier_acquire(&locks[slice_col], slice_idx);
1328 global_reduce(slice_idx == 0, last);
1329 barrier_release(&locks[slice_col], last);
1330 }
1309 - if (last) // only the last block in a slice actually writes the result
1331 + if (last) // only the last block in a slice actually writes the result
1332 write_result();
1333 slice_row = 0;
1334 slice_col_par++;
@@ -1315,13 +1337,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1337 if (slice_iters) {
1338 a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1339 (threadIdx.x % a_gl_rd_delta_o);
1318 -#pragma unroll
1340 + #pragma unroll
1341 for (int i = 0; i < b_sh_wr_iters; i++)
1342 B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1343 if (slice_col == 0) {
1322 -#pragma unroll
1323 - for (int i = 0; i < b_sh_wr_iters; i++)
1324 - B_ptr[i] -= b_gl_stride;
1344 + #pragma unroll
1345 + for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1346 }
1347
1348 // Update slice k/n for scales loading
@@ -1341,23 +1362,24 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1362 }
1363 }
1364
1344 -#define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1345 - HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1346 - else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1347 - thread_n_blocks == THREAD_N_BLOCKS && \
1348 - thread_k_blocks == THREAD_K_BLOCKS && \
1349 - has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1350 - num_threads == NUM_THREADS) { \
1351 - cudaFuncSetAttribute( \
1352 - Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1353 - THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1354 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1355 - Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1356 - THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1357 - <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1358 - A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1359 - prob_k, locks); \
1360 - }
1365 + #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1366 + THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1367 + else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1368 + thread_n_blocks == THREAD_N_BLOCKS && \
1369 + thread_k_blocks == THREAD_K_BLOCKS && \
1370 + has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1371 + num_threads == NUM_THREADS) { \
1372 + cudaFuncSetAttribute( \
1373 + Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1374 + THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1375 + GROUP_BLOCKS>, \
1376 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1377 + Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1378 + THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1379 + GROUP_BLOCKS><<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1380 + A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1381 + prob_k, locks); \
1382 + }
1383
1384 typedef struct {
1385 int thread_k;
@@ -1389,7 +1411,7 @@ thread_config_t large_batch_thread_configs[] = {
1411
1412 };
1413
1392 -int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1414 +int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
1415 int prob_n, int prob_k, int num_bits, int group_size,
1416 bool has_act_order, bool is_k_full) {
1417 bool cache_scales_chunk = has_act_order && !is_k_full;
@@ -1402,15 +1424,15 @@ int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1424 if (group_size == -1) {
1425 tb_groups = 1;
1426 } else if (group_size == 0) {
1405 - tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1427 + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1428 } else {
1429 tb_groups = div_ceil(tb_k, group_size);
1430 }
1431
1432 if (cache_scales_chunk) {
1433 int load_groups =
1412 - tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1413 - load_groups = max(load_groups, 32); // We load at least 32 scale groups
1434 + tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1435 + load_groups = max(load_groups, 32); // We load at least 32 scale groups
1436 return load_groups * tb_n * 2;
1437
1438 } else {
@@ -1420,7 +1442,7 @@ int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1442 }
1443 }
1444
1423 -bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1445 +bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
1446 int prob_m, int prob_n, int prob_k, int num_bits,
1447 int scales_cache_size, int max_shared_mem) {
1448 int pack_factor = 32 / num_bits;
@@ -1451,12 +1473,12 @@ bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1473
1474 float pipe_size = (a_size + b_size) * pipe_stages;
1475
1454 - TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1476 + TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1477
1478 return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1479 }
1480
1459 -bool is_valid_config(thread_config_t const &th_config, int max_m_blocks,
1481 +bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1482 int prob_m, int prob_n, int prob_k, int num_bits,
1483 int group_size, bool has_act_order, bool is_k_full,
1484 int max_shared_mem) {
@@ -1519,43 +1541,43 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1541 }
1542 }
1543
1522 - max_m_blocks--; // Process less M blocks per invocation to reduce cache
1523 - // usage
1544 + max_m_blocks--; // Process less M blocks per invocation to reduce cache
1545 + // usage
1546 }
1547
1548 return exec_config_t{0, {-1, -1, -1}};
1549 }
1550
1529 -#define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1530 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1531 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1532 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1533 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1534 - \
1535 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1536 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1537 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1538 - __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1539 - \
1540 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1541 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1542 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1543 - __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1544 - \
1545 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1546 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1547 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1548 - __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1549 - \
1550 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1551 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1552 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1553 - __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1551 + #define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1552 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1553 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1554 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1555 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1556 + \
1557 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1558 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1559 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1560 + __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1561 + \
1562 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1563 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1564 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1565 + __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1566 + \
1567 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1568 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1569 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1570 + __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1571 + \
1572 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1573 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1574 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1575 + __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1576
1577 template <typename scalar_t>
1556 -void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1557 - void *g_idx, void *perm, void *a_tmp, int prob_m,
1558 - int prob_n, int prob_k, void *workspace, int num_bits,
1578 +void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1579 + void* g_idx, void* perm, void* a_tmp, int prob_m,
1580 + int prob_n, int prob_k, void* workspace, int num_bits,
1581 bool has_act_order, bool is_k_full, int num_groups,
1582 int group_size, int dev, cudaStream_t stream, int thread_k,
1583 int thread_n, int sms, int max_par) {
@@ -1639,15 +1661,15 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1661 }
1662 }
1663
1642 - const int4 *A_ptr = (const int4 *)A;
1643 - const int4 *B_ptr = (const int4 *)B;
1644 - int4 *C_ptr = (int4 *)C;
1645 - const int4 *s_ptr = (const int4 *)s;
1646 - const int *g_idx_ptr = (const int *)g_idx;
1647 - const int *perm_ptr = (const int *)perm;
1648 - int4 *a_tmp_ptr = (int4 *)a_tmp;
1664 + const int4* A_ptr = (const int4*)A;
1665 + const int4* B_ptr = (const int4*)B;
1666 + int4* C_ptr = (int4*)C;
1667 + const int4* s_ptr = (const int4*)s;
1668 + const int* g_idx_ptr = (const int*)g_idx;
1669 + const int* perm_ptr = (const int*)perm;
1670 + int4* a_tmp_ptr = (int4*)a_tmp;
1671
1650 - int *locks = (int *)workspace;
1672 + int* locks = (int*)workspace;
1673
1674 if (has_act_order) {
1675 // Permute A columns
@@ -1673,8 +1695,7 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1695 // Note that parallel > 1 currently only works for inputs without any
1696 // padding
1697 par = (16 * thread_m_blocks - pad) / (16 * exec_cfg.max_m_blocks);
1676 - if (par > max_par)
1677 - par = max_par;
1698 + if (par > max_par) par = max_par;
1699 prob_m = (16 * exec_cfg.max_m_blocks) * par;
1700 i += exec_cfg.max_m_blocks * (par - 1);
1701 thread_m_blocks = exec_cfg.max_m_blocks;
@@ -1709,11 +1730,11 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1730 }
1731 }
1732
1712 -} // namespace gptq_marlin
1733 +} // namespace gptq_marlin
1734
1714 -torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1715 - torch::Tensor &b_scales, torch::Tensor &g_idx,
1716 - torch::Tensor &perm, torch::Tensor &workspace,
1735 +torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1736 + torch::Tensor& b_scales, torch::Tensor& g_idx,
1737 + torch::Tensor& perm, torch::Tensor& workspace,
1738 int64_t num_bits, int64_t size_m, int64_t size_n,
1739 int64_t size_k, bool is_k_full) {
1740 // Verify num_bits
@@ -1824,18 +1845,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1845 int dev = a.get_device();
1846 if (a.scalar_type() == at::ScalarType::Half) {
1847 gptq_marlin::marlin_mm_f16i4<half>(
1827 - a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(), b_scales.data_ptr<at::Half>(),
1828 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n,
1829 - size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1830 - num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1831 - thread_k, thread_n, sms, gptq_marlin::max_par);
1848 + a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
1849 + b_scales.data_ptr<at::Half>(), g_idx.data_ptr(), perm.data_ptr(),
1850 + a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
1851 + workspace.data_ptr(), num_bits, has_act_order, is_k_full, num_groups,
1852 + group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1853 + thread_n, sms, gptq_marlin::max_par);
1854 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1855 gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
1834 - a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(), c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1835 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n,
1836 - size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1837 - num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1838 - thread_k, thread_n, sms, gptq_marlin::max_par);
1856 + a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
1857 + c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1858 + g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(),
1859 + size_m, size_n, size_k, workspace.data_ptr(), num_bits, has_act_order,
1860 + is_k_full, num_groups, group_size, dev,
1861 + at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1862 + gptq_marlin::max_par);
1863 } else {
1864 TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
1865 }
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+28 -22
@@ -11,22 +11,23 @@
11
12 namespace gptq_marlin {
13
14 -// 8 warps are a good choice since every SM has 4 schedulers and having more than 1 warp per
15 -// schedule allows some more latency hiding. At the same time, we want relatively few warps to have
16 -// many registers per warp and small tiles.
14 +// 8 warps are a good choice since every SM has 4 schedulers and having more
15 +// than 1 warp per schedule allows some more latency hiding. At the same time,
16 +// we want relatively few warps to have many registers per warp and small tiles.
17 static constexpr int default_threads = 256;
18
19 -static constexpr int pipe_stages = 4; // 4 pipeline stages fit into shared memory
19 +static constexpr int pipe_stages =
20 + 4; // 4 pipeline stages fit into shared memory
21
22 static constexpr int min_thread_n = 64;
23 static constexpr int min_thread_k = 64;
24
25 static constexpr int tile_size = 16;
25 -static constexpr int max_par = 16;
26 +static constexpr int max_par = 16;
27
28 template <typename T, int n>
29 struct Vec {
29 - T elems[n];
30 + T elems[n];
31 __device__ T& operator[](int i) { return elems[i]; }
32 };
33
@@ -35,30 +36,35 @@ using I4 = Vec<int, 4>;
36 constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
37
38 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38 - // No support for async
39 +// No support for async
40 #else
41
41 -__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool pred = true) {
42 +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
43 + bool pred = true) {
44 const int BYTES = 16;
43 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
44 - asm volatile("{\n"
45 - " .reg .pred p;\n"
46 - " setp.ne.b32 p, %0, 0;\n"
47 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
48 - "}\n" ::"r"((int)pred),
49 - "r"(smem), "l"(glob_ptr), "n"(BYTES));
45 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
46 + asm volatile(
47 + "{\n"
48 + " .reg .pred p;\n"
49 + " setp.ne.b32 p, %0, 0;\n"
50 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
51 + "}\n" ::"r"((int)pred),
52 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
53 }
54
55 __device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
56 const int BYTES = 16;
54 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55 - asm volatile("{\n"
56 - " cp.async.cg.shared.global [%0], [%1], %2;\n"
57 - "}\n" ::"r"(smem),
58 - "l"(glob_ptr), "n"(BYTES));
57 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
58 + asm volatile(
59 + "{\n"
60 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
61 + "}\n" ::"r"(smem),
62 + "l"(glob_ptr), "n"(BYTES));
63 }
64
61 -__device__ inline void cp_async_fence() { asm volatile("cp.async.commit_group;\n" ::); }
65 +__device__ inline void cp_async_fence() {
66 + asm volatile("cp.async.commit_group;\n" ::);
67 +}
68
69 template <int n>
70 __device__ inline void cp_async_wait() {
@@ -67,4 +73,4 @@ __device__ inline void cp_async_wait() {
73
74 #endif
75
70 -} // namespace gptq_marlin
76 +} // namespace gptq_marlin
csrc/quantization/gptq_marlin/gptq_marlin_dtypes.cuh
+52 -37
@@ -5,58 +5,73 @@
5 #include <cuda_fp16.h>
6 #include <cuda_bf16.h>
7
8 -
8 namespace gptq_marlin {
9
10 template <typename scalar_t>
12 -class ScalarType {
13 -};
11 +class ScalarType {};
12
13 template <>
14 class ScalarType<half> {
17 -public:
18 - using scalar_t = half;
19 - using scalar_t2 = half2;
20 -
21 - // Matrix fragments for tensor core instructions; their precise layout is
22 - // documented here:
23 - // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
24 - using FragA = Vec<half2, 4>;
25 - using FragB = Vec<half2, 2>;
26 - using FragC = Vec<float, 4>;
27 - using FragS = Vec<half2, 1>;
28 -
29 - static __device__ float inline num2float(const half x) { return __half2float(x); }
30 -
31 - static __device__ half2 inline num2num2(const half x) { return __half2half2(x); }
32 -
33 - static __device__ half2 inline nums2num2(const half x1, const half x2) { return __halves2half2(x1, x2); }
34 -
35 - static __host__ __device__ half inline float2num(const float x) { return __float2half(x); }
15 + public:
16 + using scalar_t = half;
17 + using scalar_t2 = half2;
18 +
19 + // Matrix fragments for tensor core instructions; their precise layout is
20 + // documented here:
21 + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
22 + using FragA = Vec<half2, 4>;
23 + using FragB = Vec<half2, 2>;
24 + using FragC = Vec<float, 4>;
25 + using FragS = Vec<half2, 1>;
26 +
27 + static __device__ float inline num2float(const half x) {
28 + return __half2float(x);
29 + }
30 +
31 + static __device__ half2 inline num2num2(const half x) {
32 + return __half2half2(x);
33 + }
34 +
35 + static __device__ half2 inline nums2num2(const half x1, const half x2) {
36 + return __halves2half2(x1, x2);
37 + }
38 +
39 + static __host__ __device__ half inline float2num(const float x) {
40 + return __float2half(x);
41 + }
42 };
43
44 template <>
45 class ScalarType<nv_bfloat16> {
40 -public:
41 - using scalar_t = nv_bfloat16;
42 - using scalar_t2 = nv_bfloat162;
46 + public:
47 + using scalar_t = nv_bfloat16;
48 + using scalar_t2 = nv_bfloat162;
49
44 - using FragA = Vec<nv_bfloat162, 4>;
45 - using FragB = Vec<nv_bfloat162, 2>;
46 - using FragC = Vec<float, 4>;
47 - using FragS = Vec<nv_bfloat162, 1>;
50 + using FragA = Vec<nv_bfloat162, 4>;
51 + using FragB = Vec<nv_bfloat162, 2>;
52 + using FragC = Vec<float, 4>;
53 + using FragS = Vec<nv_bfloat162, 1>;
54
55 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
50 - static __device__ float inline num2float(const nv_bfloat16 x) { return __bfloat162float(x); }
51 -
52 - static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { return __bfloat162bfloat162(x); }
53 -
54 - static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { return __halves2bfloat162(x1, x2); }
55 -
56 - static __host__ __device__ nv_bfloat16 inline float2num(const float x) { return __float2bfloat16(x); }
56 + static __device__ float inline num2float(const nv_bfloat16 x) {
57 + return __bfloat162float(x);
58 + }
59 +
60 + static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) {
61 + return __bfloat162bfloat162(x);
62 + }
63 +
64 + static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1,
65 + const nv_bfloat16 x2) {
66 + return __halves2bfloat162(x1, x2);
67 + }
68 +
69 + static __host__ __device__ nv_bfloat16 inline float2num(const float x) {
70 + return __float2bfloat16(x);
71 + }
72 #endif
73 };
74
60 -}
75 +} // namespace gptq_marlin
76
77 #endif
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+46 -48
@@ -12,14 +12,14 @@ static constexpr int tile_n_size = tile_k_size * 4;
12 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
13
14 template <int const num_threads, int const num_bits, bool const has_perm>
15 -__global__ void
16 -marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
17 - uint32_t const *__restrict__ perm_ptr,
18 - uint32_t *__restrict__ out_ptr, int size_k, int size_n) {}
15 +__global__ void marlin_repack_kernel(
16 + uint32_t const* __restrict__ b_q_weight_ptr,
17 + uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
18 + int size_k, int size_n) {}
19
20 -} // namespace gptq_marlin
20 +} // namespace gptq_marlin
21
22 -torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
22 +torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
23 int64_t size_k, int64_t size_n,
24 int64_t num_bits) {
25 TORCH_CHECK_NOT_IMPLEMENTED(
@@ -30,10 +30,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
30 #else
31
32 template <int const num_threads, int const num_bits, bool const has_perm>
33 -__global__ void
34 -marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
35 - uint32_t const *__restrict__ perm_ptr,
36 - uint32_t *__restrict__ out_ptr, int size_k, int size_n) {
33 +__global__ void marlin_repack_kernel(
34 + uint32_t const* __restrict__ b_q_weight_ptr,
35 + uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
36 + int size_k, int size_n) {
37 constexpr int pack_factor = 32 / num_bits;
38
39 int k_tiles = size_k / tile_k_size;
@@ -61,8 +61,8 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
61
62 constexpr int perm_size = tile_k_size / 4;
63
64 - int4 *sh_perm_ptr = sh;
65 - int4 *sh_pipe_ptr = sh_perm_ptr;
64 + int4* sh_perm_ptr = sh;
65 + int4* sh_pipe_ptr = sh_perm_ptr;
66 if constexpr (has_perm) {
67 sh_pipe_ptr += perm_size;
68 }
@@ -76,7 +76,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
76 auto load_perm_to_shared = [&](int k_tile_id) {
77 int first_k_int4 = (k_tile_id * tile_k_size) / 4;
78
79 - int4 const *perm_int4_ptr = reinterpret_cast<int4 const *>(perm_ptr);
79 + int4 const* perm_int4_ptr = reinterpret_cast<int4 const*>(perm_ptr);
80
81 if (threadIdx.x < perm_size) {
82 sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x];
@@ -92,22 +92,22 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
92
93 int first_n = n_tile_id * tile_n_size;
94
95 - int4 *sh_ptr = sh_pipe_ptr + stage_size * pipe;
95 + int4* sh_ptr = sh_pipe_ptr + stage_size * pipe;
96
97 if constexpr (has_perm) {
98 if (threadIdx.x < stage_size) {
99 int k_id = threadIdx.x / stage_n_threads;
100 int n_id = threadIdx.x % stage_n_threads;
101
102 - uint32_t const *sh_perm_int_ptr =
103 - reinterpret_cast<uint32_t const *>(sh_perm_ptr);
102 + uint32_t const* sh_perm_int_ptr =
103 + reinterpret_cast<uint32_t const*>(sh_perm_ptr);
104
105 int src_k = sh_perm_int_ptr[k_id];
106 int src_k_packed = src_k / pack_factor;
107
108 cp_async4(
109 &sh_ptr[k_id * stage_n_threads + n_id],
110 - reinterpret_cast<int4 const *>(&(
110 + reinterpret_cast<int4 const*>(&(
111 b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
112 }
113
@@ -120,7 +120,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
120 int first_k_packed = first_k / pack_factor;
121
122 cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
123 - reinterpret_cast<int4 const *>(
123 + reinterpret_cast<int4 const*>(
124 &(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
125 first_n + (n_id * 4)])));
126 }
@@ -151,10 +151,10 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
151 constexpr int sh_stride = 64;
152 constexpr uint32_t mask = (1 << num_bits) - 1;
153
154 - int4 *sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155 - uint32_t *sh_stage_int_ptr = reinterpret_cast<uint32_t *>(sh_stage_ptr);
154 + int4* sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155 + uint32_t* sh_stage_int_ptr = reinterpret_cast<uint32_t*>(sh_stage_ptr);
156
157 - uint32_t *sh_perm_int_ptr = reinterpret_cast<uint32_t *>(sh_perm_ptr);
157 + uint32_t* sh_perm_int_ptr = reinterpret_cast<uint32_t*>(sh_perm_ptr);
158
159 uint32_t vals[8];
160
@@ -176,17 +176,16 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
176 }
177
178 } else {
179 -
179 uint32_t b1_vals[tile_ints];
180 uint32_t b2_vals[tile_ints];
181
183 -#pragma unroll
182 + #pragma unroll
183 for (int i = 0; i < tile_ints; i++) {
184 b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
185 b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
186 }
187
189 -#pragma unroll
188 + #pragma unroll
189 for (int i = 0; i < 4; i++) {
190 int cur_elem = tc_row + tc_offsets[i];
191 int cur_int = cur_elem / pack_factor;
@@ -206,7 +205,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
205 constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
206
207 uint32_t res = 0;
209 -#pragma unroll
208 + #pragma unroll
209 for (int i = 0; i < 8; i++) {
210 res |= vals[pack_idx[i]] << (i * 4);
211 }
@@ -218,7 +217,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
217
218 uint32_t res1 = 0;
219 uint32_t res2 = 0;
221 -#pragma unroll
220 + #pragma unroll
221 for (int i = 0; i < 4; i++) {
222 res1 |= vals[pack_idx[i]] << (i * 8);
223 res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -230,14 +229,14 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
229 };
230
231 auto start_pipes = [&](int k_tile_id, int n_tile_id) {
233 -#pragma unroll
232 + #pragma unroll
233 for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
234 fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
235 }
236
237 wait_for_stage();
238 };
240 -#pragma unroll
239 + #pragma unroll
240 for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
241 int n_tile_id = 0;
242
@@ -248,7 +247,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
247 start_pipes(k_tile_id, n_tile_id);
248
249 while (n_tile_id < n_tiles) {
251 -#pragma unroll
250 + #pragma unroll
251 for (int pipe = 0; pipe < repack_stages; pipe++) {
252 fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
253 n_tile_id + pipe + repack_stages - 1);
@@ -260,21 +259,21 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
259 }
260 }
261
263 -} // namespace gptq_marlin
264 -
265 -#define CALL_IF(NUM_BITS, HAS_PERM) \
266 - else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
267 - cudaFuncSetAttribute( \
268 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
269 - NUM_BITS, HAS_PERM>, \
270 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
271 - gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
272 - HAS_PERM> \
273 - <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
274 - b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
275 - }
262 +} // namespace gptq_marlin
263 +
264 + #define CALL_IF(NUM_BITS, HAS_PERM) \
265 + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
266 + cudaFuncSetAttribute( \
267 + gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
268 + NUM_BITS, HAS_PERM>, \
269 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
270 + gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
271 + HAS_PERM> \
272 + <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
273 + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
274 + }
275
277 -torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
276 +torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
277 int64_t size_k, int64_t size_n,
278 int64_t num_bits) {
279 // Verify compatibility with marlin tile of 16x64
@@ -318,11 +317,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
317 bool has_perm = perm.size(0) != 0;
318
319 // Get ptrs
321 - uint32_t const *b_q_weight_ptr =
322 - reinterpret_cast<uint32_t const *>(b_q_weight.data_ptr());
323 - uint32_t const *perm_ptr =
324 - reinterpret_cast<uint32_t const *>(perm.data_ptr());
325 - uint32_t *out_ptr = reinterpret_cast<uint32_t *>(out.data_ptr());
320 + uint32_t const* b_q_weight_ptr =
321 + reinterpret_cast<uint32_t const*>(b_q_weight.data_ptr());
322 + uint32_t const* perm_ptr = reinterpret_cast<uint32_t const*>(perm.data_ptr());
323 + uint32_t* out_ptr = reinterpret_cast<uint32_t*>(out.data_ptr());
324
325 // Get dev info
326 int dev = b_q_weight.get_device();
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+229 -231
@@ -25,7 +25,10 @@
25
26 #include <iostream>
27
28 -template <typename T> inline std::string str(T x) { return std::to_string(x); }
28 +template <typename T>
29 +inline std::string str(T x) {
30 + return std::to_string(x);
31 +}
32
33 namespace marlin {
34
@@ -38,9 +41,10 @@ constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
41 // corresponding index accesses must be compile-time constants, which is why we
42 // extensively use `#pragma unroll` throughout the kernel code to guarantee
43 // this.
41 -template <typename T, int n> struct Vec {
44 +template <typename T, int n>
45 +struct Vec {
46 T elems[n];
43 - __device__ T &operator[](int i) { return elems[i]; }
47 + __device__ T& operator[](int i) { return elems[i]; }
48 };
49
50 using I4 = Vec<int, 4>;
@@ -51,29 +55,32 @@ using I4 = Vec<int, 4>;
55 using FragA = Vec<half2, 4>;
56 using FragB = Vec<half2, 2>;
57 using FragC = Vec<float, 4>;
54 -using FragS = Vec<half2, 1>; // quantization scales
58 +using FragS = Vec<half2, 1>; // quantization scales
59
60 // Predicated asynchronous global->shared copy; used for inputs A where we apply
61 // predication to handle batchsizes that are not multiples of 16.
58 -__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
62 +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63 bool pred = true) {
64 const int BYTES = 16;
65 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
62 - asm volatile("{\n"
63 - " .reg .pred p;\n"
64 - " setp.ne.b32 p, %0, 0;\n"
65 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
66 - "}\n" ::"r"((int)pred),
67 - "r"(smem), "l"(glob_ptr), "n"(BYTES));
66 + asm volatile(
67 + "{\n"
68 + " .reg .pred p;\n"
69 + " setp.ne.b32 p, %0, 0;\n"
70 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71 + "}\n" ::"r"((int)pred),
72 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
73 }
74
75 // Asynchronous global->shared copy
71 -__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
76 +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77 const int BYTES = 16;
78 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
74 - asm volatile("{\n"
75 - " cp.async.cg.shared.global [%0], [%1], %2;\n"
76 - "}\n" :: "r"(smem), "l"(glob_ptr), "n"(BYTES));
79 + asm volatile(
80 + "{\n"
81 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
82 + "}\n" ::"r"(smem),
83 + "l"(glob_ptr), "n"(BYTES));
84 }
85
86 // Async copy fence.
@@ -82,28 +89,30 @@ __device__ inline void cp_async_fence() {
89 }
90
91 // Wait until at most `n` async copy stages are still pending.
85 -template <int n> __device__ inline void cp_async_wait() {
92 +template <int n>
93 +__device__ inline void cp_async_wait() {
94 asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95 }
96
97 // m16n8k16 tensor core mma instruction with fp16 inputs and fp32
98 // output/accumulation.
91 -__device__ inline void mma(const FragA &a_frag, const FragB &frag_b,
92 - FragC &frag_c) {
93 - const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
94 - const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
95 - float *c = reinterpret_cast<float *>(&frag_c);
96 - asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
97 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
98 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
99 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
100 - "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
99 +__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
100 + FragC& frag_c) {
101 + const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
102 + const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
103 + float* c = reinterpret_cast<float*>(&frag_c);
104 + asm volatile(
105 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
106 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
107 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
108 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
109 + "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
110 }
111
112 // Instruction for loading a full 16x16 matrix fragment of operand A from shared
113 // memory, directly in tensor core layout.
105 -__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
106 - uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
114 +__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
115 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
116 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
117 asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
118 : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
@@ -113,7 +122,8 @@ __device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
122 // Lookup-table based 3-input logical operation; explicitly used for
123 // dequantization as the compiler does not seem to automatically recognize it in
124 // all cases.
116 -template <int lut> __device__ inline int lop3(int a, int b, int c) {
125 +template <int lut>
126 +__device__ inline int lop3(int a, int b, int c) {
127 int res;
128 asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
129 : "=r"(res)
@@ -138,24 +148,24 @@ __device__ inline FragB dequant(int q) {
148 const int MUL = 0x2c002c00;
149 const int ADD = 0xd480d480;
150 FragB frag_b;
141 - frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
142 - *reinterpret_cast<const half2 *>(&SUB));
143 - frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
144 - *reinterpret_cast<const half2 *>(&MUL),
145 - *reinterpret_cast<const half2 *>(&ADD));
151 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152 + *reinterpret_cast<const half2*>(&SUB));
153 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
154 + *reinterpret_cast<const half2*>(&MUL),
155 + *reinterpret_cast<const half2*>(&ADD));
156 return frag_b;
157 }
158
159 // Multiply dequantized values by the corresponding quantization scale; used
160 // only for grouped quantization.
151 -__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
152 - half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
161 +__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
162 + half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
163 frag_b[0] = __hmul2(frag_b[0], s);
164 frag_b[1] = __hmul2(frag_b[1], s);
165 }
166
167 // Wait until barrier reaches `count`, then lock for current threadblock.
158 -__device__ inline void barrier_acquire(int *lock, int count) {
168 +__device__ inline void barrier_acquire(int* lock, int count) {
169 if (threadIdx.x == 0) {
170 int state = -1;
171 do
@@ -170,7 +180,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
180 }
181
182 // Release barrier and increment visitation count.
173 -__device__ inline void barrier_release(int *lock, bool reset = false) {
183 +__device__ inline void barrier_release(int* lock, bool reset = false) {
184 __syncthreads();
185 if (threadIdx.x == 0) {
186 if (reset) {
@@ -187,26 +197,27 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
197 }
198 }
199
190 -template <const int threads, // number of threads in a threadblock
191 - const int thread_m_blocks, // number of 16x16 blocks in the m
192 - // dimension (batchsize) of the threadblock
193 - const int thread_n_blocks, // same for n dimension (output)
194 - const int thread_k_blocks, // same for k dimension (reduction)
195 - const int stages, // number of stages for the async global->shared
196 - // fetch pipeline
197 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
198 - // a separate quantization scale
200 +template <const int threads, // number of threads in a threadblock
201 + const int thread_m_blocks, // number of 16x16 blocks in the m
202 + // dimension (batchsize) of the
203 + // threadblock
204 + const int thread_n_blocks, // same for n dimension (output)
205 + const int thread_k_blocks, // same for k dimension (reduction)
206 + const int stages, // number of stages for the async global->shared
207 + // fetch pipeline
208 + const int group_blocks = -1 // number of consecutive 16x16 blocks
209 + // with a separate quantization scale
210 >
200 -__global__ void
201 -Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
202 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
203 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
204 - const int4
205 - *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
206 - int prob_m, // batch dimension m
207 - int prob_n, // output dimension n
208 - int prob_k, // reduction dimension k
209 - int *locks // extra global storage for barrier synchronization
211 +__global__ void Marlin(
212 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
213 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
214 + int4* __restrict__ C, // fp16 output buffer of shape mxn
215 + const int4* __restrict__ s, // fp16 quantization scales of shape
216 + // (k/groupsize)xn
217 + int prob_m, // batch dimension m
218 + int prob_n, // output dimension n
219 + int prob_k, // reduction dimension k
220 + int* locks // extra global storage for barrier synchronization
221 ) {
222 // Each threadblock processes one "stripe" of the B matrix with (roughly) the
223 // same size, which might involve multiple column "slices" (of width 16 *
@@ -241,11 +252,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
252 int slice_row = (iters * blockIdx.x) % k_tiles;
253 int slice_col_par = (iters * blockIdx.x) / k_tiles;
254 int slice_col = slice_col_par;
244 - int slice_iters; // number of threadblock tiles in the current slice
255 + int slice_iters; // number of threadblock tiles in the current slice
256 int slice_count =
246 - 0; // total number of active threadblocks in the current slice
247 - int slice_idx; // index of threadblock in current slice; numbered bottom to
248 - // top
257 + 0; // total number of active threadblocks in the current slice
258 + int slice_idx; // index of threadblock in current slice; numbered bottom to
259 + // top
260
261 // We can easily implement parallel problem execution by just remapping
262 // indices and advancing global pointers
@@ -261,27 +272,22 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
272 auto init_slice = [&]() {
273 slice_iters =
274 iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
264 - if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
265 - slice_iters = 0;
266 - if (slice_iters == 0)
267 - return;
268 - if (slice_row + slice_iters > k_tiles)
269 - slice_iters = k_tiles - slice_row;
275 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
276 + if (slice_iters == 0) return;
277 + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
278 slice_count = 1;
279 slice_idx = 0;
280 int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
281 if (col_first <= k_tiles * (slice_col_par + 1)) {
282 int col_off = col_first - k_tiles * slice_col_par;
283 slice_count = ceildiv(k_tiles - col_off, iters);
276 - if (col_off > 0)
277 - slice_count++;
284 + if (col_off > 0) slice_count++;
285 int delta_first = iters * blockIdx.x - col_first;
286 if (delta_first < 0 || (col_off == 0 && delta_first == 0))
287 slice_idx = slice_count - 1;
288 else {
289 slice_idx = slice_count - 1 - delta_first / iters;
283 - if (col_off > 0)
284 - slice_idx--;
290 + if (col_off > 0) slice_idx--;
291 }
292 }
293 if (slice_col == n_tiles) {
@@ -293,29 +299,30 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
299 };
300 init_slice();
301
296 - int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
302 + int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
303 // We typically use `constexpr` to indicate that this value is a compile-time
304 // constant
305 constexpr int a_sh_stride =
300 - 16 * thread_k_blocks / 8; // stride of an A matrix tile in shared memory
306 + 16 * thread_k_blocks / 8; // stride of an A matrix tile in shared memory
307 constexpr int a_gl_rd_delta_o =
308 16 * thread_k_blocks /
303 - 8; // delta between subsequent A tiles in global memory
309 + 8; // delta between subsequent A tiles in global memory
310 int a_gl_rd_delta_i =
311 a_gl_stride *
306 - (threads / a_gl_rd_delta_o); // between subsequent accesses within a tile
312 + (threads / a_gl_rd_delta_o); // between subsequent accesses within a tile
313 constexpr int a_sh_wr_delta =
308 - a_sh_stride * (threads / a_gl_rd_delta_o); // between shared memory writes
314 + a_sh_stride *
315 + (threads / a_gl_rd_delta_o); // between shared memory writes
316 constexpr int a_sh_rd_delta_o =
317 2 * ((threads / 32) /
311 - (thread_n_blocks / 4)); // between shared memory tile reads
318 + (thread_n_blocks / 4)); // between shared memory tile reads
319 constexpr int a_sh_rd_delta_i =
313 - a_sh_stride * 16; // within a shared memory tile
320 + a_sh_stride * 16; // within a shared memory tile
321 constexpr int a_sh_stage =
315 - a_sh_stride * (16 * thread_m_blocks); // overall size of a tile
322 + a_sh_stride * (16 * thread_m_blocks); // overall size of a tile
323 constexpr int a_sh_wr_iters =
324 ceildiv(a_sh_stage,
318 - a_sh_wr_delta); // number of shared write iterations for a tile
325 + a_sh_wr_delta); // number of shared write iterations for a tile
326
327 int b_gl_stride = 16 * prob_n / 32;
328 constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
@@ -368,7 +375,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
375 // needed if there are more threads than required for a certain tilesize or
376 // when the batchsize is not a multiple of 16.
377 bool a_sh_wr_pred[a_sh_wr_iters];
371 -#pragma unroll
378 + #pragma unroll
379 for (int i = 0; i < a_sh_wr_iters; i++)
380 a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
381 bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -387,13 +394,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
394 // loop unrolls, all shared memory accesses are static, we simply precompute
395 // both transformed reads and writes.
396 int a_sh_wr_trans[a_sh_wr_iters];
390 -#pragma unroll
397 + #pragma unroll
398 for (int i = 0; i < a_sh_wr_iters; i++)
399 a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
400 int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
394 -#pragma unroll
401 + #pragma unroll
402 for (int i = 0; i < b_sh_wr_iters; i++) {
396 -#pragma unroll
403 + #pragma unroll
404 for (int j = 0; j < thread_m_blocks; j++)
405 a_sh_rd_trans[i][j] =
406 transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -403,16 +410,16 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
410 // runtime; we break dependencies between subsequent accesses with a tile by
411 // maintining multiple pointers (we have enough registers), a tiny
412 // optimization.
406 - const int4 *B_ptr[b_sh_wr_iters];
407 -#pragma unroll
413 + const int4* B_ptr[b_sh_wr_iters];
414 + #pragma unroll
415 for (int i = 0; i < b_sh_wr_iters; i++)
416 B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
417
418 extern __shared__ int4 sh[];
419 // Shared memory storage for global fetch pipelines.
413 - int4 *sh_a = sh;
414 - int4 *sh_b = sh_a + (stages * a_sh_stage);
415 - int4 *sh_s = sh_b + (stages * b_sh_stage);
420 + int4* sh_a = sh;
421 + int4* sh_b = sh_a + (stages * a_sh_stage);
422 + int4* sh_s = sh_b + (stages * b_sh_stage);
423 // Register storage for double buffer of shared memory reads.
424 FragA frag_a[2][thread_m_blocks];
425 I4 frag_b_quant[2];
@@ -421,34 +428,33 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
428
429 // Zero accumulators.
430 auto zero_accums = [&]() {
424 -#pragma unroll
431 + #pragma unroll
432 for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
426 - reinterpret_cast<float *>(frag_c)[i] = 0;
433 + reinterpret_cast<float*>(frag_c)[i] = 0;
434 };
435
436 // Asynchronously fetch the next A, B and s tile from global to the next
437 // shared memory pipeline location.
438 auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
439 if (pred) {
433 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
434 -#pragma unroll
440 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
441 + #pragma unroll
442 for (int i = 0; i < a_sh_wr_iters; i++) {
443 cp_async4_pred(
444 &sh_a_stage[a_sh_wr_trans[i]],
445 &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
446 a_sh_wr_pred[i]);
447 }
441 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
442 -#pragma unroll
448 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
449 + #pragma unroll
450 for (int i = 0; i < b_sh_wr_iters; i++) {
451 cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
452 B_ptr[i] += b_gl_rd_delta_o;
453 }
454 // Only fetch scales if this tile starts a new group
455 if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
449 - int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
450 - if (s_sh_wr_pred)
451 - cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
456 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
457 + if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
458 s_gl_rd += s_gl_rd_delta;
459 }
460 }
@@ -475,37 +481,35 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
481 // theoretically better attempts have lead to bad instruction ordering by
482 // the compiler and correspondingly a noticeable drop in performance.
483 if (group_blocks != -1) {
478 - int4 *sh_s_stage =
484 + int4* sh_s_stage =
485 sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
486 (pipe / (group_blocks / thread_k_blocks)));
481 - reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
487 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
488 }
483 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
484 -#pragma unroll
489 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
490 + #pragma unroll
491 for (int i = 0; i < thread_m_blocks; i++)
492 ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
487 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
488 - frag_b_quant[k % 2] = *reinterpret_cast<I4 *>(
493 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
494 + frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
495 &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
496 };
497
498 // Execute the actual tensor core matmul of a sub-tile.
499 auto matmul = [&](int k) {
494 -// We have the m dimension as the inner loop in order to encourage overlapping
495 -// dequantization and matmul operations.
496 -#pragma unroll
500 + // We have the m dimension as the inner loop in order to encourage overlapping
501 + // dequantization and matmul operations.
502 + #pragma unroll
503 for (int j = 0; j < 4; j++) {
504 int b_quant = frag_b_quant[k % 2][j];
505 int b_quant_shift = b_quant >> 8;
506 FragB frag_b0 = dequant(b_quant);
507 // If there are no groups, we can just scale the final output once and can
508 // avoid doing so for each weight.
503 - if (group_blocks != -1)
504 - scale(frag_b0, frag_s[k % 2][j], 0);
509 + if (group_blocks != -1) scale(frag_b0, frag_s[k % 2][j], 0);
510 FragB frag_b1 = dequant(b_quant_shift);
506 - if (group_blocks != -1)
507 - scale(frag_b1, frag_s[k % 2][j], 1);
508 -#pragma unroll
511 + if (group_blocks != -1) scale(frag_b1, frag_s[k % 2][j], 1);
512 + #pragma unroll
513 for (int i = 0; i < thread_m_blocks; i++) {
514 mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
515 mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
@@ -530,38 +534,38 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
534 // unnecessary read or write iterations, e.g., for two warps we write only
535 // once by warp 1 and read only once by warp 0.
536
533 -#pragma unroll
537 + #pragma unroll
538 for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
535 -#pragma unroll
539 + #pragma unroll
540 for (int i = red_off; i > 0; i /= 2) {
541 if (i <= red_idx && red_idx < 2 * i) {
538 -#pragma unroll
542 + #pragma unroll
543 for (int j = 0; j < 4 * 2; j++) {
544 int red_sh_wr =
545 red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
546 if (i < red_off) {
543 - float *c_rd = reinterpret_cast<float *>(
544 - &sh[red_sh_delta * j + red_sh_rd]);
545 - float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
546 -#pragma unroll
547 + float* c_rd =
548 + reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
549 + float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
550 + #pragma unroll
551 for (int k = 0; k < 4; k++)
548 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
552 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
553 c_rd[k] + c_wr[k];
554 }
555 sh[red_sh_wr] =
552 - reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
556 + reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
557 }
558 }
559 __syncthreads();
560 }
561 if (red_idx == 0) {
558 -#pragma unroll
562 + #pragma unroll
563 for (int i = 0; i < 4 * 2; i++) {
560 - float *c_rd =
561 - reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
562 -#pragma unroll
564 + float* c_rd =
565 + reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
566 + #pragma unroll
567 for (int j = 0; j < 4; j++)
564 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
568 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
569 c_rd[j];
570 }
571 }
@@ -571,9 +575,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
575 };
576
577 // Since multiple threadblocks may process parts of the same column slice, we
574 - // finally have to globally reduce over the results. As the striped partitioning
575 - // minimizes the number of such reductions and our outputs are usually rather
576 - // small, we perform this reduction serially in L2 cache.
578 + // finally have to globally reduce over the results. As the striped
579 + // partitioning minimizes the number of such reductions and our outputs are
580 + // usually rather small, we perform this reduction serially in L2 cache.
581 auto global_reduce = [&](bool first = false, bool last = false) {
582 // We are very careful here to reduce directly in the output buffer to
583 // maximize L2 cache utilization in this step. To do this, we write out
@@ -592,39 +596,39 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
596 int row = (threadIdx.x % 32) / 4;
597
598 if (!first) {
595 -// Interestingly, doing direct global accesses here really seems to mess up the
596 -// compiler and lead to slowdowns, hence we also use async-copies even though
597 -// these fetches are not actually asynchronous.
598 -#pragma unroll
599 + // Interestingly, doing direct global accesses here really seems to mess up
600 + // the compiler and lead to slowdowns, hence we also use async-copies even
601 + // though these fetches are not actually asynchronous.
602 + #pragma unroll
603 for (int i = 0; i < thread_m_blocks * 4; i++) {
600 - cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
601 - &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
602 - c_gl_wr_delta_i * (i % 2)],
603 - i < (thread_m_blocks - 1) * 4 ||
604 - 8 * (i / 2) + row < prob_m);
604 + cp_async4_pred(
605 + &sh[c_sh_wr + c_sh_wr_delta * i],
606 + &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
607 + c_gl_wr_delta_i * (i % 2)],
608 + i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
609 }
610 cp_async_fence();
611 cp_async_wait<0>();
612 }
613
610 -#pragma unroll
614 + #pragma unroll
615 for (int i = 0; i < thread_m_blocks * 4; i++) {
616 if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
617 if (!first) {
618 int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
615 -#pragma unroll
619 + #pragma unroll
620 for (int j = 0; j < 2 * 4; j++) {
617 - reinterpret_cast<float *>(
621 + reinterpret_cast<float*>(
622 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
619 - __half2float(reinterpret_cast<__half *>(&c_red)[j]);
623 + __half2float(reinterpret_cast<__half*>(&c_red)[j]);
624 }
625 }
626 if (!last) {
627 int4 c;
624 -#pragma unroll
628 + #pragma unroll
629 for (int j = 0; j < 2 * 4; j++) {
626 - reinterpret_cast<__half *>(&c)[j] =
627 - __float2half(reinterpret_cast<float *>(
630 + reinterpret_cast<__half*>(&c)[j] =
631 + __float2half(reinterpret_cast<float*>(
632 &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
633 }
634 C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -658,17 +662,17 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
662
663 // We first reorder in shared memory to guarantee the most efficient final
664 // global write patterns
661 - auto write = [&](int idx, float c0, float c1, FragS &s) {
665 + auto write = [&](int idx, float c0, float c1, FragS& s) {
666 half2 res = __halves2half2(__float2half(c0), __float2half(c1));
667 if (group_blocks ==
664 - -1) // for per-column quantization we finally apply the scale here
668 + -1) // for per-column quantization we finally apply the scale here
669 res = __hmul2(res, s[0]);
666 - ((half2 *)sh)[idx] = res;
670 + ((half2*)sh)[idx] = res;
671 };
672 if (threadIdx.x / 32 < thread_n_blocks / 4) {
669 -#pragma unroll
673 + #pragma unroll
674 for (int i = 0; i < thread_m_blocks; i++) {
671 -#pragma unroll
675 + #pragma unroll
676 for (int j = 0; j < 4; j++) {
677 int wr = c_sh_wr + 8 * j;
678 write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
@@ -685,7 +689,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
689 }
690 __syncthreads();
691
688 -#pragma unroll
692 + #pragma unroll
693 for (int i = 0;
694 i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
695 i++) {
@@ -699,9 +703,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
703
704 // Start global fetch and register load pipelines.
705 auto start_pipes = [&]() {
702 -#pragma unroll
703 - for (int i = 0; i < stages - 1; i++)
704 - fetch_to_shared(i, i, i < slice_iters);
706 + #pragma unroll
707 + for (int i = 0; i < stages - 1; i++) fetch_to_shared(i, i, i < slice_iters);
708 zero_accums();
709 wait_for_stage();
710 fetch_to_registers(0, 0);
@@ -711,12 +714,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
714
715 // Main loop.
716 while (slice_iters) {
714 -// We unroll over both the global fetch and the register load pipeline to ensure
715 -// all shared memory accesses are static. Note that both pipelines have even
716 -// length meaning that the next iteration will always start at index 0.
717 -#pragma unroll
717 + // We unroll over both the global fetch and the register load pipeline to
718 + // ensure all shared memory accesses are static. Note that both pipelines have
719 + // even length meaning that the next iteration will always start at index 0.
720 + #pragma unroll
721 for (int pipe = 0; pipe < stages;) {
719 -#pragma unroll
722 + #pragma unroll
723 for (int k = 0; k < b_sh_wr_iters; k++) {
724 fetch_to_registers(k + 1, pipe % stages);
725 if (k == b_sh_wr_iters - 2) {
@@ -728,8 +731,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
731 matmul(k);
732 }
733 slice_iters--;
731 - if (slice_iters == 0)
732 - break;
734 + if (slice_iters == 0) break;
735 }
736 a_gl_rd += a_gl_rd_delta_o * stages;
737
@@ -742,8 +744,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
744 // For per-column scales, we only fetch them here in the final step before
745 // write-out
746 if (group_blocks == -1 && last) {
745 - if (s_sh_wr_pred)
746 - cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
747 + if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
748 cp_async_fence();
749 }
750 thread_block_reduce();
@@ -751,17 +752,17 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
752 cp_async_wait<0>();
753 __syncthreads();
754 if (threadIdx.x / 32 < thread_n_blocks / 4) {
754 - reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
755 - reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
755 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
756 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
757 }
758 }
758 - if (slice_count > 1) { // only globally reduce if there is more than one
759 - // block in a slice
759 + if (slice_count > 1) { // only globally reduce if there is more than one
760 + // block in a slice
761 barrier_acquire(&locks[slice_col], slice_idx);
762 global_reduce(slice_idx == 0, last);
763 barrier_release(&locks[slice_col], last);
764 }
764 - if (last) // only the last block in a slice actually writes the result
765 + if (last) // only the last block in a slice actually writes the result
766 write_result();
767 slice_row = 0;
768 slice_col_par++;
@@ -770,13 +771,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
771 if (slice_iters) {
772 a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
773 (threadIdx.x % a_gl_rd_delta_o);
773 -#pragma unroll
774 + #pragma unroll
775 for (int i = 0; i < b_sh_wr_iters; i++)
776 B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
777 if (slice_col == 0) {
777 -#pragma unroll
778 - for (int i = 0; i < b_sh_wr_iters; i++)
779 - B_ptr[i] -= b_gl_stride;
778 + #pragma unroll
779 + for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
780 }
781 s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
782 start_pipes();
@@ -787,26 +787,27 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
787
788 #else
789
790 -template <const int threads, // number of threads in a threadblock
791 - const int thread_m_blocks, // number of 16x16 blocks in the m
792 - // dimension (batchsize) of the threadblock
793 - const int thread_n_blocks, // same for n dimension (output)
794 - const int thread_k_blocks, // same for k dimension (reduction)
795 - const int stages, // number of stages for the async global->shared
796 - // fetch pipeline
797 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
798 - // a separate quantization scale
790 +template <const int threads, // number of threads in a threadblock
791 + const int thread_m_blocks, // number of 16x16 blocks in the m
792 + // dimension (batchsize) of the
793 + // threadblock
794 + const int thread_n_blocks, // same for n dimension (output)
795 + const int thread_k_blocks, // same for k dimension (reduction)
796 + const int stages, // number of stages for the async global->shared
797 + // fetch pipeline
798 + const int group_blocks = -1 // number of consecutive 16x16 blocks
799 + // with a separate quantization scale
800 >
800 -__global__ void
801 -Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
802 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
803 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
804 - const int4
805 - *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
806 - int prob_m, // batch dimension m
807 - int prob_n, // output dimension n
808 - int prob_k, // reduction dimension k
809 - int *locks // extra global storage for barrier synchronization
801 +__global__ void Marlin(
802 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
803 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
804 + int4* __restrict__ C, // fp16 output buffer of shape mxn
805 + const int4* __restrict__ s, // fp16 quantization scales of shape
806 + // (k/groupsize)xn
807 + int prob_m, // batch dimension m
808 + int prob_n, // output dimension n
809 + int prob_k, // reduction dimension k
810 + int* locks // extra global storage for barrier synchronization
811 ) {
812 // Marlin is not implemented yet for SM < 8.0
813 assert(false);
@@ -819,10 +820,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
820 // than 1 warp per schedule allows some more latency hiding. At the same time,
821 // we want relatively few warps to have many registers per warp and small tiles.
822 const int USER_THREADS =
822 - 256; // Note: This is only used with user-provided thread_k/n
823 -const int STAGES = 4; // 4 pipeline stages fit into shared memory
823 + 256; // Note: This is only used with user-provided thread_k/n
824 +const int STAGES = 4; // 4 pipeline stages fit into shared memory
825 const int SHARED_MEM =
825 - 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
826 + 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
827
828 static constexpr int min_thread_n = 64;
829 static constexpr int min_thread_k = 64;
@@ -831,7 +832,7 @@ static constexpr int tile_size = 16;
832 static constexpr int max_par = 16;
833
834 static constexpr int pack_factor_4bit =
834 - 8; // We have 8 4-bit vals inside a 32 bit
835 + 8; // We have 8 4-bit vals inside a 32 bit
836
837 #define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
838 GROUP_BLOCKS, NUM_THREADS) \
@@ -858,23 +859,23 @@ thread_config_t small_batch_thread_configs[] = {
859 // Ordered by priority
860
861 // thread_k, thread_n, num_threads
861 - {128, 128, 256}, // Default
862 - {128, 64, 128}, // Reduce N 2X, same K
863 - {64, 256, 256}, // Reduce K 2X, increase N 2X
864 - {64, 128, 128}, // Reduce K 2X, same N
862 + {128, 128, 256}, // Default
863 + {128, 64, 128}, // Reduce N 2X, same K
864 + {64, 256, 256}, // Reduce K 2X, increase N 2X
865 + {64, 128, 128}, // Reduce K 2X, same N
866 };
867
868 thread_config_t large_batch_thread_configs[] = {
869 // Ordered by priority
870
871 // thread_k, thread_n, num_threads
871 - {64, 256, 256}, // Default
872 - {128, 128, 256}, // Reduce N 2X, increase K 2X
873 - {64, 128, 128}, // Reduce N 2X, same K
874 - {128, 64, 128}, // Reduce N 4X, increase K 2X
872 + {64, 256, 256}, // Default
873 + {128, 128, 256}, // Reduce N 2X, increase K 2X
874 + {64, 128, 128}, // Reduce N 2X, same K
875 + {128, 64, 128}, // Reduce N 4X, increase K 2X
876 };
877
877 -bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
878 +bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
879 int prob_k) {
880 // Sanity
881 if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
@@ -907,7 +908,6 @@ bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
908 }
909
910 thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
910 -
911 if (prob_m <= 16) {
912 for (auto th_config : small_batch_thread_configs) {
913 if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
@@ -926,20 +926,20 @@ thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
926 return thread_config_t{-1, -1, -1};
927 }
928
929 -#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
930 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
931 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
932 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
933 - __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
934 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
935 - __CALL_IF(2, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
936 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
937 - __CALL_IF(3, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
938 - __CALL_IF(4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
929 +#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
930 + __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
931 + __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
932 + __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
933 + __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
934 + __CALL_IF(2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
935 + __CALL_IF(2, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
936 + __CALL_IF(3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
937 + __CALL_IF(3, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
938 + __CALL_IF(4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
939 __CALL_IF(4, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS)
940
941 -void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
942 - int prob_n, int prob_k, void *workspace, int groupsize = -1,
941 +void marlin_cuda(const void* A, const void* B, void* C, void* s, int prob_m,
942 + int prob_n, int prob_k, void* workspace, int groupsize = -1,
943 int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
944 int thread_n = -1, int sms = -1, int max_par = 16) {
945 int tot_m = prob_m;
@@ -996,12 +996,12 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
996 " is not divisible by group_blocks = ", group_blocks);
997 }
998
999 - const int4 *A_ptr = (const int4 *)A;
1000 - const int4 *B_ptr = (const int4 *)B;
1001 - int4 *C_ptr = (int4 *)C;
1002 - const int4 *s_ptr = (const int4 *)s;
999 + const int4* A_ptr = (const int4*)A;
1000 + const int4* B_ptr = (const int4*)B;
1001 + int4* C_ptr = (int4*)C;
1002 + const int4* s_ptr = (const int4*)s;
1003
1004 - int *locks = (int *)workspace;
1004 + int* locks = (int*)workspace;
1005
1006 for (int i = 0; i < tot_m_blocks; i += 4) {
1007 int thread_m_blocks = tot_m_blocks - i;
@@ -1011,8 +1011,7 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
1011 // Note that parallel > 1 currently only works for inputs without any
1012 // padding
1013 par = (16 * thread_m_blocks - pad) / 64;
1014 - if (par > max_par)
1015 - par = max_par;
1014 + if (par > max_par) par = max_par;
1015 prob_m = 64 * par;
1016 i += 4 * (par - 1);
1017 thread_m_blocks = 4;
@@ -1041,12 +1040,11 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
1040 }
1041 }
1042
1044 -} // namespace marlin
1043 +} // namespace marlin
1044
1046 -torch::Tensor marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1047 - torch::Tensor &b_scales, torch::Tensor &workspace,
1045 +torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1046 + torch::Tensor& b_scales, torch::Tensor& workspace,
1047 int64_t size_m, int64_t size_n, int64_t size_k) {
1049 -
1048 // Verify M
1049 TORCH_CHECK(size_m == a.size(0),
1050 "Shape mismatch: a.size(0) = " + str(a.size(0)) +
@@ -1074,9 +1072,9 @@ torch::Tensor marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1072
1073 int actual_size_n =
1074 (b_q_weight.size(1) / marlin::tile_size) * marlin::pack_factor_4bit;
1077 - TORCH_CHECK(size_n == actual_size_n,
1078 - "size_n = " + str(size_n) +
1079 - ", actual_size_n = " + str(actual_size_n));
1075 + TORCH_CHECK(
1076 + size_n == actual_size_n,
1077 + "size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
1078
1079 // Verify A device and strides
1080 TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
csrc/quantization/marlin/sparse/common/base.h
+7 -5
@@ -26,12 +26,14 @@ constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
26 // corresponding index accesses must be compile-time constants, which is why we
27 // extensively use `#pragma unroll` throughout the kernel code to guarantee
28 // this.
29 -template <typename T, int n> struct Vec {
29 +template <typename T, int n>
30 +struct Vec {
31 T elems[n];
31 - __device__ T &operator[](int i) { return elems[i]; }
32 + __device__ T& operator[](int i) { return elems[i]; }
33 };
34
34 -template <int M_, int N_, int K_> struct ShapeBase {
35 +template <int M_, int N_, int K_>
36 +struct ShapeBase {
37 static constexpr int M = M_, N = N_, K = K_;
38 };
39
@@ -44,6 +46,6 @@ using FragA = Vec<half2, 4>;
46 using FragB = Vec<half2, 2>;
47 using FragM = Vec<uint, 1>;
48 using FragC = Vec<float, 4>;
47 -using FragS = Vec<half2, 1>; // quantization scales
49 +using FragS = Vec<half2, 1>; // quantization scales
50
49 -} // namespace marlin_24
51 +} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mem.h
+34 -30
@@ -21,41 +21,44 @@
21 namespace marlin_24 {
22 // Predicated asynchronous global->shared copy; used for inputs A where we apply
23 // predication to handle batchsizes that are not multiples of 16.
24 -__device__ inline void cp_async4_pred_zfill(void *smem_ptr,
25 - const void *glob_ptr,
24 +__device__ inline void cp_async4_pred_zfill(void* smem_ptr,
25 + const void* glob_ptr,
26 bool pred = true,
27 const bool zfill = false) {
28 const int BYTES = 16;
29 int src_in_bytes = (zfill ? 0 : BYTES);
30 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
31 - asm volatile("{\n"
32 - " .reg .pred p;\n"
33 - " setp.ne.b32 p, %0, 0;\n"
34 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
35 - "}\n" ::"r"((int)pred),
36 - "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
31 + asm volatile(
32 + "{\n"
33 + " .reg .pred p;\n"
34 + " setp.ne.b32 p, %0, 0;\n"
35 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
36 + "}\n" ::"r"((int)pred),
37 + "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
38 }
39
39 -__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
40 +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
41 bool pred = true) {
42 const int BYTES = 16;
43 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
43 - asm volatile("{\n"
44 - " .reg .pred p;\n"
45 - " setp.ne.b32 p, %0, 0;\n"
46 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
47 - "}\n" ::"r"((int)pred),
48 - "r"(smem), "l"(glob_ptr), "n"(BYTES));
44 + asm volatile(
45 + "{\n"
46 + " .reg .pred p;\n"
47 + " setp.ne.b32 p, %0, 0;\n"
48 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
49 + "}\n" ::"r"((int)pred),
50 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
51 }
52
53 // Asynchronous global->shared copy
52 -__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
54 +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
55 const int BYTES = 16;
56 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55 - asm volatile("{\n"
56 - " cp.async.cg.shared.global [%0], [%1], %2;\n"
57 - "}\n" ::"r"(smem),
58 - "l"(glob_ptr), "n"(BYTES));
57 + asm volatile(
58 + "{\n"
59 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
60 + "}\n" ::"r"(smem),
61 + "l"(glob_ptr), "n"(BYTES));
62 }
63
64 // Async copy fence.
@@ -64,22 +67,23 @@ __device__ inline void cp_async_fence() {
67 }
68
69 // Wait until at most `n` async copy stages are still pending.
67 -template <int n> __device__ inline void cp_async_wait() {
70 +template <int n>
71 +__device__ inline void cp_async_wait() {
72 asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
73 }
74
75 // Instruction for loading a full 16x16 matrix fragment of operand A from shared
76 // memory, directly in tensor core layout.
73 -__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
74 - uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
77 +__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
78 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
79 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
80 asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
81 : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
82 : "r"(smem));
83 }
84
81 -__device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
82 - uint32_t *a = reinterpret_cast<uint32_t *>(&frag_m);
85 +__device__ inline void ldsm4_m(FragM& frag_m, const void* smem_ptr) {
86 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_m);
87 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
88 asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n"
89 : "=r"(a[0]), "=r"(a[1])
@@ -88,8 +92,8 @@ __device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
92
93 // Instruction for loading a full 16x16 matrix fragment of operand A from shared
94 // memory, directly in tensor core layout.
91 -__device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
92 - uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
95 +__device__ inline void ldsm4_t(FragA& frag_a, const void* smem_ptr) {
96 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
97 uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
98 asm volatile(
99 "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
@@ -98,7 +102,7 @@ __device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
102 }
103
104 // Wait until barrier reaches `count`, then lock for current threadblock.
101 -__device__ inline void barrier_acquire(int *lock, int count) {
105 +__device__ inline void barrier_acquire(int* lock, int count) {
106 if (threadIdx.x == 0) {
107 int state = -1;
108 do
@@ -113,7 +117,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
117 }
118
119 // Release barrier and increment visitation count.
116 -__device__ inline void barrier_release(int *lock, bool reset = false) {
120 +__device__ inline void barrier_release(int* lock, bool reset = false) {
121 __syncthreads();
122 if (threadIdx.x == 0) {
123 if (reset) {
@@ -129,4 +133,4 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
133 : "l"(lock), "r"(val));
134 }
135 }
132 -} // namespace marlin_24
136 +} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mma.h
+56 -51
@@ -22,51 +22,56 @@ namespace marlin_24 {
22
23 // m16n8k32 sparse tensor core mma instruction with fp16 inputs and fp32
24 // output/accumulation.
25 -__device__ inline void mma_sp(const FragB &a_frag0, const FragB &a_frag1,
26 - const FragA &frag_b, FragC &frag_c, FragM &frag_m,
25 +__device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
26 + const FragA& frag_b, FragC& frag_c, FragM& frag_m,
27 const int psel) {
28 - const uint32_t *a0 = reinterpret_cast<const uint32_t *>(&a_frag0);
29 - const uint32_t *a1 = reinterpret_cast<const uint32_t *>(&a_frag1);
30 - const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
31 - const uint32_t *e = reinterpret_cast<const uint32_t *>(&frag_m);
32 - float *c = reinterpret_cast<float *>(&frag_c);
28 + const uint32_t* a0 = reinterpret_cast<const uint32_t*>(&a_frag0);
29 + const uint32_t* a1 = reinterpret_cast<const uint32_t*>(&a_frag1);
30 + const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
31 + const uint32_t* e = reinterpret_cast<const uint32_t*>(&frag_m);
32 + float* c = reinterpret_cast<float*>(&frag_c);
33 if (psel == 0) {
34 - asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
35 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
36 - "{%12,%13,%14,%15}, %16, 0x0;\n"
37 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
38 - : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
39 - "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
40 - "f"(c[2]), "f"(c[3]), "r"(e[0]));
41 - asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
42 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
43 - "{%12,%13,%14,%15}, %16, 0x0;\n"
44 - : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
45 - : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
46 - "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
47 - "f"(c[6]), "f"(c[7]), "r"(e[0]));
34 + asm volatile(
35 + "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
36 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
37 + "{%12,%13,%14,%15}, %16, 0x0;\n"
38 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
39 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
40 + "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
41 + "r"(e[0]));
42 + asm volatile(
43 + "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
44 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
45 + "{%12,%13,%14,%15}, %16, 0x0;\n"
46 + : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
47 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
48 + "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
49 + "r"(e[0]));
50 } else {
49 - asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
50 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
51 - "{%12,%13,%14,%15}, %16, 0x1;\n"
52 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
53 - : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
54 - "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
55 - "f"(c[2]), "f"(c[3]), "r"(e[0]));
56 - asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
57 - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
58 - "{%12,%13,%14,%15}, %16, 0x1;\n"
59 - : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
60 - : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
61 - "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
62 - "f"(c[6]), "f"(c[7]), "r"(e[0]));
51 + asm volatile(
52 + "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
53 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
54 + "{%12,%13,%14,%15}, %16, 0x1;\n"
55 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
56 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
57 + "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
58 + "r"(e[0]));
59 + asm volatile(
60 + "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
61 + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
62 + "{%12,%13,%14,%15}, %16, 0x1;\n"
63 + : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
64 + : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
65 + "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
66 + "r"(e[0]));
67 }
68 }
69
70 // Lookup-table based 3-input logical operation; explicitly used for
71 // dequantization as the compiler does not seem to automatically recognize it in
72 // all cases.
69 -template <int lut> __device__ inline int lop3(int a, int b, int c) {
73 +template <int lut>
74 +__device__ inline int lop3(int a, int b, int c) {
75 int res;
76 asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
77 : "=r"(res)
@@ -120,11 +125,11 @@ __device__ inline FragB dequant_4bit(int q) {
125 const int ADD = 0xd480d480;
126
127 FragB frag_b;
123 - frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
124 - *reinterpret_cast<const half2 *>(&SUB));
125 - frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
126 - *reinterpret_cast<const half2 *>(&MUL),
127 - *reinterpret_cast<const half2 *>(&ADD));
128 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
129 + *reinterpret_cast<const half2*>(&SUB));
130 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
131 + *reinterpret_cast<const half2*>(&MUL),
132 + *reinterpret_cast<const half2*>(&ADD));
133 return frag_b;
134 }
135
@@ -143,24 +148,24 @@ __device__ inline FragB dequant_8bit(int q) {
148 static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
149
150 FragB frag_b;
146 - frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
147 - *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
148 - frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
149 - *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
151 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
153 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
154 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
155 return frag_b;
156 }
157
158 // Multiply dequantized values by the corresponding quantization scale; used
159 // only for grouped quantization.
155 -__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
156 - half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
160 +__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
161 + half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
162 frag_b[0] = __hmul2(frag_b[0], s);
163 frag_b[1] = __hmul2(frag_b[1], s);
164 }
165
161 -__device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
162 - FragS &s0, float *c4, float *c5, float *c6,
163 - float *c7, FragS &s1) {
166 +__device__ inline void scale_floats(float* c0, float* c1, float* c2, float* c3,
167 + FragS& s0, float* c4, float* c5, float* c6,
168 + float* c7, FragS& s1) {
169 *c0 = __fmul_rn(*c0, __half2float(s0[0].x));
170 *c1 = __fmul_rn(*c1, __half2float(s0[0].y));
171 *c2 = __fmul_rn(*c2, __half2float(s0[1].x));
@@ -172,4 +177,4 @@ __device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
177 *c7 = __fmul_rn(*c7, __half2float(s1[1].y));
178 }
179
175 -} // namespace marlin_24
180 +} // namespace marlin_24
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+218 -228
@@ -32,12 +32,15 @@
32
33 #else
34
35 -#include "common/mem.h"
36 -#include "common/mma.h"
35 + #include "common/mem.h"
36 + #include "common/mma.h"
37
38 #endif
39
40 -template <typename T> inline std::string str(T x) { return std::to_string(x); }
40 +template <typename T>
41 +inline std::string str(T x) {
42 + return std::to_string(x);
43 +}
44
45 namespace marlin_24 {
46
@@ -45,7 +48,7 @@ namespace marlin_24 {
48 // than 1 warp per schedule allows some more latency hiding. At the same time,
49 // we want relatively few warps to have many registers per warp and small tiles.
50 static constexpr int THREADS = 256;
48 -static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
51 +static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
52
53 static constexpr int min_thread_n = 128;
54
@@ -54,35 +57,36 @@ static constexpr int max_par = 16;
57
58 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
59
57 -template <const int num_bits, // weight bits
58 - const int threads, // number of threads in a threadblock
59 - const int thread_m_blocks, // number of 16x16 blocks in the m
60 - // dimension (batchsize) of the threadblock
61 - const int thread_n_blocks, // same for n dimension (output)
62 - const int thread_k_blocks, // same for k dimension (reduction)
63 - const int stages, // number of stages for the async global->shared
64 - // fetch pipeline
65 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
66 - // a separate quantization scale
60 +template <const int num_bits, // weight bits
61 + const int threads, // number of threads in a threadblock
62 + const int thread_m_blocks, // number of 16x16 blocks in the m
63 + // dimension (batchsize) of the
64 + // threadblock
65 + const int thread_n_blocks, // same for n dimension (output)
66 + const int thread_k_blocks, // same for k dimension (reduction)
67 + const int stages, // number of stages for the async global->shared
68 + // fetch pipeline
69 + const int group_blocks = -1 // number of consecutive 16x16 blocks
70 + // with a separate quantization scale
71 >
72 __global__ void Marlin_24(
69 - const int4 *__restrict__ A, // fp16 input matrix of shape mxk
70 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
71 - const int4
72 - *__restrict__ meta, // 2bit metadata information about 2:4 format on B
73 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
74 - const int4
75 - *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
76 - int prob_m, // batch dimension m
77 - int prob_n, // output dimension n
78 - int prob_k, // reduction dimension k
79 - int *locks // extra global storage for barrier synchronization
73 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
74 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
75 + const int4* __restrict__ meta, // 2bit metadata information about 2:4
76 + // format on B
77 + int4* __restrict__ C, // fp16 output buffer of shape mxn
78 + const int4* __restrict__ s, // fp16 quantization scales of shape
79 + // (k/groupsize)xn
80 + int prob_m, // batch dimension m
81 + int prob_n, // output dimension n
82 + int prob_k, // reduction dimension k
83 + int* locks // extra global storage for barrier synchronization
84 ) {}
85
82 -torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
83 - torch::Tensor &b_meta,
84 - torch::Tensor &b_scales,
85 - torch::Tensor &workspace, int64_t num_bits,
86 +torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
87 + torch::Tensor& b_meta,
88 + torch::Tensor& b_scales,
89 + torch::Tensor& workspace, int64_t num_bits,
90 int64_t size_m, int64_t size_n,
91 int64_t size_k) {
92 TORCH_CHECK_NOT_IMPLEMENTED(
@@ -92,29 +96,30 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
96
97 #else
98
95 -template <const int num_bits, // weight bits
96 - const int threads, // number of threads in a threadblock
97 - const int thread_m_blocks, // number of 16x16 blocks in the m
98 - // dimension (batchsize) of the threadblock
99 - const int thread_n_blocks, // same for n dimension (output)
100 - const int thread_k_blocks, // same for k dimension (reduction)
101 - const int stages, // number of stages for the async global->shared
102 - // fetch pipeline
103 - const int group_blocks = -1 // number of consecutive 16x16 blocks with
104 - // a separate quantization scale
99 +template <const int num_bits, // weight bits
100 + const int threads, // number of threads in a threadblock
101 + const int thread_m_blocks, // number of 16x16 blocks in the m
102 + // dimension (batchsize) of the
103 + // threadblock
104 + const int thread_n_blocks, // same for n dimension (output)
105 + const int thread_k_blocks, // same for k dimension (reduction)
106 + const int stages, // number of stages for the async global->shared
107 + // fetch pipeline
108 + const int group_blocks = -1 // number of consecutive 16x16 blocks
109 + // with a separate quantization scale
110 >
111 __global__ void Marlin_24(
107 - const int4 *__restrict__ A, // fp16 input matrix of shape mxk
108 - const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
109 - const int4
110 - *__restrict__ meta, // 2bit metadata information about 2:4 format on B
111 - int4 *__restrict__ C, // fp16 output buffer of shape mxn
112 - const int4
113 - *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
114 - int prob_m, // batch dimension m
115 - int prob_n, // output dimension n
116 - int prob_k, // reduction dimension k
117 - int *locks // extra global storage for barrier synchronization
112 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
113 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
114 + const int4* __restrict__ meta, // 2bit metadata information about 2:4
115 + // format on B
116 + int4* __restrict__ C, // fp16 output buffer of shape mxn
117 + const int4* __restrict__ s, // fp16 quantization scales of shape
118 + // (k/groupsize)xn
119 + int prob_m, // batch dimension m
120 + int prob_n, // output dimension n
121 + int prob_k, // reduction dimension k
122 + int* locks // extra global storage for barrier synchronization
123 ) {
124 // Each threadblock processes one "stripe" of the B matrix with (roughly) the
125 // same size, which might involve multiple column "slices" (of width 16 *
@@ -174,27 +179,22 @@ __global__ void Marlin_24(
179 auto init_slice = [&]() {
180 slice_iters =
181 iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
177 - if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
178 - slice_iters = 0;
179 - if (slice_iters == 0)
180 - return;
181 - if (slice_row + slice_iters > k_tiles)
182 - slice_iters = k_tiles - slice_row;
182 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
183 + if (slice_iters == 0) return;
184 + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
185 slice_count = 1;
186 slice_idx = 0;
187 int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
188 if (col_first <= k_tiles * (slice_col_par + 1)) {
189 int col_off = col_first - k_tiles * slice_col_par;
190 slice_count = ceildiv(k_tiles - col_off, iters);
189 - if (col_off > 0)
190 - slice_count++;
191 + if (col_off > 0) slice_count++;
192 int delta_first = iters * blockIdx.x - col_first;
193 if (delta_first < 0 || (col_off == 0 && delta_first == 0))
194 slice_idx = slice_count - 1;
195 else {
196 slice_idx = slice_count - 1 - delta_first / iters;
196 - if (col_off > 0)
197 - slice_idx--;
197 + if (col_off > 0) slice_idx--;
198 }
199 }
200 if (slice_col == n_tiles) {
@@ -207,7 +207,7 @@ __global__ void Marlin_24(
207 init_slice();
208
209 // RLC: 8 is vec_size -> 128-bit instructions, 8 fp16 elements
210 - int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
210 + int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
211
212 // stride of an A matrix tile in shared memory
213 constexpr int a_sh_stride = 32 * thread_k_blocks / 8;
@@ -239,9 +239,9 @@ __global__ void Marlin_24(
239 constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
240 constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
241
242 - int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
242 + int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
243 constexpr int m_sh_stride =
244 - (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
244 + (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
245 int m_gl_rd_delta_o = m_gl_stride * thread_k_blocks;
246 int m_gl_rd_delta_i = m_gl_stride * (threads / m_sh_stride);
247 constexpr int m_sh_wr_delta = threads / 2;
@@ -305,7 +305,7 @@ __global__ void Marlin_24(
305 // needed if there are more threads than required for a certain tilesize or
306 // when the batchsize is not a multiple of 16.
307 bool a_sh_wr_pred[a_sh_wr_iters];
308 -#pragma unroll
308 + #pragma unroll
309 for (int i = 0; i < a_sh_wr_iters; i++) {
310 a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
311 }
@@ -325,13 +325,13 @@ __global__ void Marlin_24(
325 // loop unrolls, all shared memory accesses are static, we simply precompute
326 // both transformed reads and writes.
327 int a_sh_wr_trans[a_sh_wr_iters];
328 -#pragma unroll
328 + #pragma unroll
329 for (int i = 0; i < a_sh_wr_iters; i++)
330 a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
331 int a_sh_rd_trans[2][b_sh_wr_iters][thread_m_blocks];
332 -#pragma unroll
332 + #pragma unroll
333 for (int i = 0; i < b_sh_wr_iters; i++) {
334 -#pragma unroll
334 + #pragma unroll
335 for (int j = 0; j < thread_m_blocks; j++) {
336 a_sh_rd_trans[0][i][j] =
337 transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -344,23 +344,23 @@ __global__ void Marlin_24(
344 // runtime; we break dependencies between subsequent accesses with a tile by
345 // maintining multiple pointers (we have enough registers), a tiny
346 // optimization.
347 - const int4 *B_ptr[b_sh_wr_iters];
348 -#pragma unroll
347 + const int4* B_ptr[b_sh_wr_iters];
348 + #pragma unroll
349 for (int i = 0; i < b_sh_wr_iters; i++)
350 B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
351
352 bool m_sh_wr_pred = threadIdx.x < m_sh_wr_delta;
353 - const int4 *meta_ptr[m_sh_iters];
354 -#pragma unroll
353 + const int4* meta_ptr[m_sh_iters];
354 + #pragma unroll
355 for (int i = 0; i < m_sh_iters; i++)
356 meta_ptr[i] = meta + m_gl_rd_delta_i * i + m_gl_rd;
357
358 extern __shared__ int4 sh[];
359 // Shared memory storage for global fetch pipelines.
360 - int4 *sh_a = sh;
361 - int4 *sh_b = sh_a + (stages * a_sh_stage);
362 - int4 *sh_s = sh_b + (stages * b_sh_stage);
363 - int4 *sh_m = sh_s + (stages * s_sh_stage);
360 + int4* sh_a = sh;
361 + int4* sh_b = sh_a + (stages * a_sh_stage);
362 + int4* sh_s = sh_b + (stages * b_sh_stage);
363 + int4* sh_m = sh_s + (stages * s_sh_stage);
364 // Register storage for double buffer of shared memory reads.
365 FragA frag_a[2][thread_m_blocks][2];
366 I4 frag_b_quant[2][b_thread_vecs];
@@ -370,46 +370,43 @@ __global__ void Marlin_24(
370
371 // Zero accumulators.
372 auto zero_accums = [&]() {
373 -#pragma unroll
373 + #pragma unroll
374 for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
375 - reinterpret_cast<float *>(frag_c)[i] = 0;
375 + reinterpret_cast<float*>(frag_c)[i] = 0;
376 };
377
378 // Asynchronously fetch the next A, B and s tile from global to the next
379 // shared memory pipeline location.
380 auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
381 if (pred) {
382 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
383 -#pragma unroll
382 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
383 + #pragma unroll
384 for (int i = 0; i < a_sh_wr_iters; i++) {
385 cp_async4_pred(
386 &sh_a_stage[a_sh_wr_trans[i]],
387 &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
388 a_sh_wr_pred[i]);
389 }
390 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
391 -#pragma unroll
390 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
391 + #pragma unroll
392 for (int i = 0; i < b_sh_wr_iters; i++) {
393 -#pragma unroll
393 + #pragma unroll
394 for (int j = 0; j < b_thread_vecs; j++) {
395 - cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j],
396 - B_ptr[i] + j);
395 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
396 }
397 B_ptr[i] += b_gl_rd_delta_o;
398 }
400 - int4 *sh_meta_stage = sh_m + m_sh_stage * pipe;
401 -#pragma unroll
399 + int4* sh_meta_stage = sh_m + m_sh_stage * pipe;
400 + #pragma unroll
401 for (int i = 0; i < m_sh_iters; i++) {
402 if (m_sh_wr_pred)
404 - cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr],
405 - meta_ptr[i]);
403 + cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr], meta_ptr[i]);
404 meta_ptr[i] += m_gl_rd_delta_o;
405 }
406 // Only fetch scales if this tile starts a new group
407 if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
410 - int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
411 - if (s_sh_wr_pred)
412 - cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
408 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
409 + if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
410 s_gl_rd += s_gl_rd_delta;
411 }
412 }
@@ -436,13 +433,13 @@ __global__ void Marlin_24(
433 // theoretically better attempts have lead to bad instruction ordering by
434 // the compiler and correspondingly a noticeable drop in performance.
435 if (group_blocks != -1) {
439 - int4 *sh_s_stage =
436 + int4* sh_s_stage =
437 sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
438 (pipe / (group_blocks / thread_k_blocks)));
442 - reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
439 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
440 }
444 - int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
445 -#pragma unroll
441 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
442 + #pragma unroll
443 for (int i = 0; i < thread_m_blocks; i++) {
444 ldsm4(frag_a[k % 2][i][0],
445 &sh_a_stage[a_sh_rd_trans[0][k % b_sh_wr_iters][i]]);
@@ -450,24 +447,24 @@ __global__ void Marlin_24(
447 &sh_a_stage[a_sh_rd_trans[1][k % b_sh_wr_iters][i]]);
448 }
449
453 - int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
454 -#pragma unroll
450 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
451 + #pragma unroll
452 for (int i = 0; i < b_thread_vecs; i++) {
456 - frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
453 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
454 &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
455 }
456
457 // Load meta with ldsm4
461 - int4 *sh_m_stage = sh_m + m_sh_stage * pipe;
458 + int4* sh_m_stage = sh_m + m_sh_stage * pipe;
459 ldsm4_m(frag_m[k % 2][0],
460 &sh_m_stage[m_sh_rd_delta * (k % m_sh_iters) + m_sh_rd]);
461 };
462
463 // Execute the actual tensor core matmul of a sub-tile.
464 auto matmul = [&](int k) {
468 -// We have the m dimension as the inner loop in order to encourage overlapping
469 -// dequantization and matmul operations.
470 -#pragma unroll
465 + // We have the m dimension as the inner loop in order to encourage overlapping
466 + // dequantization and matmul operations.
467 + #pragma unroll
468 for (int j = 0; j < 4; j++) {
469 FragB frag_b0;
470 FragB frag_b1;
@@ -480,7 +477,7 @@ __global__ void Marlin_24(
477 frag_b1 = dequant_4bit(b_quant_shift);
478
479 } else {
483 - int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
480 + int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
481 int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
482 int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
483
@@ -497,7 +494,7 @@ __global__ void Marlin_24(
494 scale(frag_b1, frag_s[k % 2][j], 1);
495 }
496
500 -#pragma unroll
497 + #pragma unroll
498 for (int i = 0; i < thread_m_blocks; i++) {
499 mma_sp(frag_b0, frag_b1, frag_a[k % 2][i][0], frag_c[i][j][0],
500 frag_m[k % 2][j / 2], j % 2);
@@ -518,41 +515,41 @@ __global__ void Marlin_24(
515 int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
516 (threadIdx.x % b_sh_stride_threads);
517
521 -// Parallel logarithmic shared memory reduction. We make sure to avoid any
522 -// unnecessary read or write iterations, e.g., for two warps we write only once
523 -// by warp 1 and read only once by warp 0.
524 -#pragma unroll
518 + // Parallel logarithmic shared memory reduction. We make sure to avoid any
519 + // unnecessary read or write iterations, e.g., for two warps we write only
520 + // once by warp 1 and read only once by warp 0.
521 + #pragma unroll
522 for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
526 -#pragma unroll
523 + #pragma unroll
524 for (int i = red_off; i > 0; i /= 2) {
525 if (i <= red_idx && red_idx < 2 * i) {
529 -#pragma unroll
526 + #pragma unroll
527 for (int j = 0; j < 4 * 2; j++) {
528 int red_sh_wr =
529 red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
530 if (i < red_off) {
534 - float *c_rd = reinterpret_cast<float *>(
535 - &sh[red_sh_delta * j + red_sh_rd]);
536 - float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
537 -#pragma unroll
531 + float* c_rd =
532 + reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
533 + float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
534 + #pragma unroll
535 for (int k = 0; k < 4; k++)
539 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
536 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
537 c_rd[k] + c_wr[k];
538 }
539 sh[red_sh_wr] =
543 - reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
540 + reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
541 }
542 }
543 __syncthreads();
544 }
545 if (red_idx == 0) {
549 -#pragma unroll
546 + #pragma unroll
547 for (int i = 0; i < 4 * 2; i++) {
551 - float *c_rd =
552 - reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
553 -#pragma unroll
548 + float* c_rd =
549 + reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
550 + #pragma unroll
551 for (int j = 0; j < 4; j++)
555 - reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
552 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
553 c_rd[j];
554 }
555 }
@@ -562,9 +559,9 @@ __global__ void Marlin_24(
559 };
560
561 // Since multiple threadblocks may process parts of the same column slice, we
565 - // finally have to globally reduce over the results. As the striped partitioning
566 - // minimizes the number of such reductions and our outputs are usually rather
567 - // small, we perform this reduction serially in L2 cache.
562 + // finally have to globally reduce over the results. As the striped
563 + // partitioning minimizes the number of such reductions and our outputs are
564 + // usually rather small, we perform this reduction serially in L2 cache.
565 auto global_reduce = [&](bool first = false, bool last = false) {
566 // We are very careful here to reduce directly in the output buffer to
567 // maximize L2 cache utilization in this step. To do this, we write out
@@ -574,7 +571,7 @@ __global__ void Marlin_24(
571 int c_gl_stride = prob_n / 8;
572 int c_gl_wr_delta_o = 2 * 4 * c_gl_stride;
573 int c_gl_wr_delta_i =
577 - c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
574 + c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
575 int c_gl_wr = 2 * c_gl_stride * (threadIdx.x % 4) +
576 8 * (threadIdx.x / 32) + (threadIdx.x % 32) / 4;
577 c_gl_wr += (2 * thread_n_blocks) * slice_col;
@@ -584,10 +581,10 @@ __global__ void Marlin_24(
581 int col = 2 * ((threadIdx.x % 32) % 4);
582
583 if (!first) {
587 -// Interestingly, doing direct global accesses here really seems to mess up the
588 -// compiler and lead to slowdowns, hence we also use async-copies even though
589 -// these fetches are not actually asynchronous.
590 -#pragma unroll
584 + // Interestingly, doing direct global accesses here really seems to mess up
585 + // the compiler and lead to slowdowns, hence we also use async-copies even
586 + // though these fetches are not actually asynchronous.
587 + #pragma unroll
588 for (int i = 0; i < thread_m_blocks * 4; i++) {
589 cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
590 &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
@@ -599,32 +596,32 @@ __global__ void Marlin_24(
596 cp_async_wait<0>();
597 }
598
602 -#pragma unroll
599 + #pragma unroll
600 for (int i = 0; i < thread_m_blocks * 4; i++) {
601 if (i < (thread_m_blocks - 1) * 4 ||
602 8 * (i / 2) + col + (i % 2) < prob_m) {
603 if (!first) {
604 int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
608 -#pragma unroll
605 + #pragma unroll
606 for (int j2 = 0; j2 < 2; j2++) {
610 -#pragma unroll
607 + #pragma unroll
608 for (int j1 = 0; j1 < 4; j1++) {
612 - reinterpret_cast<float *>(
609 + reinterpret_cast<float*>(
610 &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
611 4 * ((i % 4) / 2) + i % 2] +=
612 __half2float(
616 - reinterpret_cast<__half *>(&c_red)[(j2 * 4 + j1)]);
613 + reinterpret_cast<__half*>(&c_red)[(j2 * 4 + j1)]);
614 }
615 }
616 }
617 if (!last) {
618 int4 c;
622 -#pragma unroll
619 + #pragma unroll
620 for (int j2 = 0; j2 < 2; j2++) {
624 -#pragma unroll
621 + #pragma unroll
622 for (int j1 = 0; j1 < 4; j1++) {
626 - reinterpret_cast<__half *>(&c)[(j2 * 4 + j1)] =
627 - __float2half(reinterpret_cast<float *>(
623 + reinterpret_cast<__half*>(&c)[(j2 * 4 + j1)] =
624 + __float2half(reinterpret_cast<float*>(
625 &frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
626 4 * ((i % 4) / 2) + i % 2]);
627 }
@@ -643,9 +640,9 @@ __global__ void Marlin_24(
640 auto write_result = [&]() {
641 int c_gl_stride = prob_n / 8;
642
646 - constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
647 - constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
648 - constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
643 + constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
644 + constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
645 + constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
646
647 int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
648
@@ -654,22 +651,22 @@ __global__ void Marlin_24(
651 c_gl_wr += (2 * thread_n_blocks) * slice_col;
652
653 int c_sh_wr = c_sh_stride_2 * ((threadIdx.x % 32) % 4) +
657 - ((threadIdx.x % 32) / 4); // RLC:
658 - c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
654 + ((threadIdx.x % 32) / 4); // RLC:
655 + c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
656
657 constexpr int c_sh_rd_delta =
661 - c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
658 + c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
659 int c_sh_rd = c_sh_stride_3 * (threadIdx.x / (2 * 2 * thread_n_blocks)) +
660 (threadIdx.x % (2 * 2 * thread_n_blocks));
661
662 int c_gl_wr_end = c_gl_stride * prob_m;
663
667 - auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS &s0,
668 - float c4, float c5, float c6, float c7, FragS &s1) {
664 + auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS& s0,
665 + float c4, float c5, float c6, float c7, FragS& s1) {
666 uint2 res[2];
667 res[0] = to_half4(c0, c1, c2, c3);
668 res[1] = to_half4(c4, c5, c6, c7);
672 - half2 *tmp = (half2 *)&res;
669 + half2* tmp = (half2*)&res;
670 // for per-column quantization we finally apply the scale here
671 if constexpr (group_blocks == -1 && num_bits == 4) {
672 tmp[0] = __hmul2(tmp[0], s0[0]);
@@ -677,12 +674,12 @@ __global__ void Marlin_24(
674 tmp[2] = __hmul2(tmp[2], s1[0]);
675 tmp[3] = __hmul2(tmp[3], s1[1]);
676 }
680 - ((int4 *)sh)[idx] = *((int4 *)&res[0]);
677 + ((int4*)sh)[idx] = *((int4*)&res[0]);
678 };
679
680 // RLC: only warp 0 and 1 baseline example
681 if (threadIdx.x / 32 < thread_n_blocks / 4) {
685 -#pragma unroll
682 + #pragma unroll
683 for (int i = 0; i < thread_m_blocks; i++) {
684 int wr = c_sh_wr;
685 write(wr, frag_c[i][0][0][0], frag_c[i][1][0][0], frag_c[i][2][0][0],
@@ -707,7 +704,7 @@ __global__ void Marlin_24(
704 }
705 __syncthreads();
706
710 -#pragma unroll
707 + #pragma unroll
708 for (int i = 0;
709 i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
710 i++) {
@@ -721,9 +718,8 @@ __global__ void Marlin_24(
718
719 // Start global fetch and register load pipelines.
720 auto start_pipes = [&]() {
724 -#pragma unroll
725 - for (int i = 0; i < stages - 1; i++)
726 - fetch_to_shared(i, i, i < slice_iters);
721 + #pragma unroll
722 + for (int i = 0; i < stages - 1; i++) fetch_to_shared(i, i, i < slice_iters);
723 zero_accums();
724 wait_for_stage();
725 fetch_to_registers(0, 0);
@@ -733,10 +729,10 @@ __global__ void Marlin_24(
729
730 // Main loop.
731 while (slice_iters) {
736 -// We unroll over both the global fetch and the register load pipeline to ensure
737 -// all shared memory accesses are static. Note that both pipelines have even
738 -// length meaning that the next iteration will always start at index 0.
739 -#pragma unroll
732 + // We unroll over both the global fetch and the register load pipeline to
733 + // ensure all shared memory accesses are static. Note that both pipelines have
734 + // even length meaning that the next iteration will always start at index 0.
735 + #pragma unroll
736 for (int pipe = 0; pipe < stages;) {
737 fetch_to_shared((pipe + stages - 1) % stages, pipe,
738 slice_iters >= stages);
@@ -747,8 +743,7 @@ __global__ void Marlin_24(
743
744 pipe++;
745 slice_iters--;
750 - if (slice_iters == 0)
751 - break;
746 + if (slice_iters == 0) break;
747 }
748 a_gl_rd += a_gl_rd_delta_o * stages;
749
@@ -762,13 +757,11 @@ __global__ void Marlin_24(
757 // write-out
758 if constexpr (group_blocks == -1) {
759 if constexpr (num_bits == 8) {
765 - if (s_sh_wr_pred)
766 - cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
760 + if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
761 cp_async_fence();
762 } else {
763 if (last) {
770 - if (s_sh_wr_pred)
771 - cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
764 + if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
765 cp_async_fence();
766 }
767 }
@@ -780,14 +773,14 @@ __global__ void Marlin_24(
773 cp_async_wait<0>();
774 __syncthreads();
775 if (threadIdx.x / 32 < thread_n_blocks / 4) {
783 - *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
776 + *(float4*)(frag_s) = *(float4*)(&sh_s[s_sh_rd]);
777 }
778 } else {
779 if (last) {
780 cp_async_wait<0>();
781 __syncthreads();
782 if (threadIdx.x / 32 < thread_n_blocks / 4) {
790 - *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
783 + *(float4*)(frag_s) = *(float4*)(&sh_s[s_sh_rd]);
784 }
785 }
786 }
@@ -798,7 +791,7 @@ __global__ void Marlin_24(
791 // overflow in fp16)
792 if constexpr (group_blocks == -1 && num_bits == 8) {
793 if (threadIdx.x / 32 < thread_n_blocks / 4) {
801 -#pragma unroll
794 + #pragma unroll
795 for (int i = 0; i < thread_m_blocks; i++) {
796 scale_floats(&frag_c[i][0][0][0], &frag_c[i][1][0][0],
797 &frag_c[i][2][0][0], &frag_c[i][3][0][0], frag_s[0][0],
@@ -827,13 +820,13 @@ __global__ void Marlin_24(
820 }
821 }
822
830 - if (slice_count > 1) { // only globally reduce if there is more than one
831 - // block in a slice
823 + if (slice_count > 1) { // only globally reduce if there is more than one
824 + // block in a slice
825 barrier_acquire(&locks[slice_col], slice_idx);
826 global_reduce(slice_idx == 0, last);
827 barrier_release(&locks[slice_col], last);
828 }
836 - if (last) // only the last block in a slice actually writes the result
829 + if (last) // only the last block in a slice actually writes the result
830 write_result();
831
832 slice_row = 0;
@@ -843,19 +836,17 @@ __global__ void Marlin_24(
836 if (slice_iters) {
837 a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
838 (threadIdx.x % a_gl_rd_delta_o);
846 -#pragma unroll
839 + #pragma unroll
840 for (int i = 0; i < b_sh_wr_iters; i++)
841 B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
849 -#pragma unroll
842 + #pragma unroll
843 for (int i = 0; i < m_sh_iters; i++)
844 meta_ptr[i] += (m_sh_stride)-m_gl_rd_delta_o * k_tiles;
845 if (slice_col == 0) {
853 -#pragma unroll
854 - for (int i = 0; i < b_sh_wr_iters; i++)
855 - B_ptr[i] -= b_gl_stride;
856 -#pragma unroll
857 - for (int i = 0; i < m_sh_iters; i++)
858 - meta_ptr[i] -= m_gl_stride;
846 + #pragma unroll
847 + for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
848 + #pragma unroll
849 + for (int i = 0; i < m_sh_iters; i++) meta_ptr[i] -= m_gl_stride;
850 }
851 s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
852 start_pipes();
@@ -866,26 +857,26 @@ __global__ void Marlin_24(
857
858 #endif
859
869 -#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
870 - THREAD_K_BLOCKS, GROUP_BLOCKS) \
871 - else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
872 - thread_n_blocks == THREAD_N_BLOCKS && \
873 - thread_k_blocks == THREAD_K_BLOCKS && \
874 - group_blocks == GROUP_BLOCKS) { \
875 - cudaFuncSetAttribute( \
876 - Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
877 - THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
878 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
879 - Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
880 - THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
881 - <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
882 - C_ptr, s_ptr, prob_n, \
883 - prob_m, prob_k, locks); \
860 +#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
861 + THREAD_K_BLOCKS, GROUP_BLOCKS) \
862 + else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
863 + thread_n_blocks == THREAD_N_BLOCKS && \
864 + thread_k_blocks == THREAD_K_BLOCKS && \
865 + group_blocks == GROUP_BLOCKS) { \
866 + cudaFuncSetAttribute( \
867 + Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
868 + THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
869 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
870 + Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
871 + THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
872 + <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
873 + C_ptr, s_ptr, prob_n, \
874 + prob_m, prob_k, locks); \
875 }
876
886 -void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
887 - void *s, int prob_m, int prob_n, int prob_k,
888 - void *workspace, int num_bits, int groupsize = -1,
877 +void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
878 + void* s, int prob_m, int prob_n, int prob_k,
879 + void* workspace, int num_bits, int groupsize = -1,
880 int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
881 int thread_m = -1, int sms = -1, int max_par = 16) {
882 int tot_n = prob_n;
@@ -904,8 +895,8 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
895
896 if (thread_k == -1 || thread_m == -1) {
897 if (prob_n <= 16) {
907 - // For small batchizes, better partitioningif is slightly more important than
908 - // better compute utilization
898 + // For small batchizes, better partitioningif is slightly more important
899 + // than better compute utilization
900 thread_k = 128;
901 thread_m = 128;
902 } else {
@@ -914,7 +905,7 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
905 }
906 }
907
917 - int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
908 + int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
909 int thread_m_blocks = thread_m / 16;
910 int group_blocks = (groupsize == -1) ? -1 : groupsize / 16;
911 int blocks = sms;
@@ -931,13 +922,13 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
922 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
923 ", ", prob_n, ", ", prob_k, "]");
924
934 - const int4 *A_ptr = (const int4 *)A;
935 - const int4 *B_ptr = (const int4 *)B;
936 - const int4 *meta_ptr = (const int4 *)meta;
937 - int4 *C_ptr = (int4 *)C;
938 - const int4 *s_ptr = (const int4 *)s;
925 + const int4* A_ptr = (const int4*)A;
926 + const int4* B_ptr = (const int4*)B;
927 + const int4* meta_ptr = (const int4*)meta;
928 + int4* C_ptr = (int4*)C;
929 + const int4* s_ptr = (const int4*)s;
930
940 - int *locks = (int *)workspace;
931 + int* locks = (int*)workspace;
932 for (int i = 0; i < tot_n_blocks; i += 4) {
933 int thread_n_blocks = tot_n_blocks - i;
934 prob_n = tot_n - 16 * i;
@@ -946,8 +937,7 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
937 // Note that parallel > 1 currently only works for inputs without any
938 // padding
939 par = (16 * thread_n_blocks - pad) / 64;
949 - if (par > max_par)
950 - par = max_par;
940 + if (par > max_par) par = max_par;
941 prob_n = 64 * par;
942 i += 4 * (par - 1);
943 thread_n_blocks = 4;
@@ -956,16 +946,16 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
946 // For compilation speed, we only define the kernel configurations that have
947 // seemed useful (in terms of performance) in our testing, however many more
948 // are, in principle, possible.
959 -
949 +
950 // the false is start of the CALL_IF macros
961 - if (false) {
962 - } // BMxBNxBK, group
951 + if (false) {
952 + } // BMxBNxBK, group
953 // 4-bit
964 - CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
965 - CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
966 - CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
967 - CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
968 - CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
954 + CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
955 + CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
956 + CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
957 + CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
958 + CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
959 CALL_IF_2_4(4, 16, 2, 2, 4)
960 CALL_IF_2_4(4, 16, 3, 2, -1)
961 CALL_IF_2_4(4, 16, 3, 2, 4)
@@ -973,11 +963,11 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
963 CALL_IF_2_4(4, 16, 4, 2, 4)
964
965 // 8-bit
976 - CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
977 - CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
978 - CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
979 - CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
980 - CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
966 + CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
967 + CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
968 + CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
969 + CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
970 + CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
971 CALL_IF_2_4(8, 16, 2, 2, 4)
972 CALL_IF_2_4(8, 16, 3, 2, -1)
973 CALL_IF_2_4(8, 16, 3, 2, 4)
@@ -997,12 +987,12 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
987 }
988 }
989
1000 -} // namespace marlin_24
990 +} // namespace marlin_24
991
1002 -torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1003 - torch::Tensor &b_meta,
1004 - torch::Tensor &b_scales,
1005 - torch::Tensor &workspace, int64_t num_bits,
992 +torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
993 + torch::Tensor& b_meta,
994 + torch::Tensor& b_scales,
995 + torch::Tensor& workspace, int64_t num_bits,
996 int64_t size_m, int64_t size_n,
997 int64_t size_k) {
998 // Verify num_bits
@@ -1037,9 +1027,9 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1027 " is not divisible by tile_size = " + str(marlin_24::tile_size));
1028
1029 int actual_size_n = (b_q_weight.size(1) / marlin_24::tile_size) * pack_factor;
1040 - TORCH_CHECK(size_n == actual_size_n,
1041 - "size_n = " + str(size_n) +
1042 - ", actual_size_n = " + str(actual_size_n));
1030 + TORCH_CHECK(
1031 + size_n == actual_size_n,
1032 + "size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
1033
1034 // Verify meta
1035 TORCH_CHECK(b_meta.size(0) == size_k / 8 / 2 / 2,
@@ -1081,7 +1071,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1071 ", is not divisible by b_scales.size(0) = " +
1072 str(b_scales.size(0)));
1073 groupsize = size_k / b_scales.size(0);
1084 - groupsize /= 2; // Because of 24
1074 + groupsize /= 2; // Because of 24
1075 }
1076
1077 // Verify groupsize
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+27 -36
@@ -22,27 +22,23 @@ __device__ inline unsigned int as_unsigned(int i) {
22 // 4-bit matvec kernel (LUT-based)
23 __global__ void NUQ4MatMulKernel(
24 #ifndef USE_ROCM
25 - const half2* __restrict__ vec,
25 + const half2* __restrict__ vec,
26 #else
27 - const __half2* __restrict__ vec,
27 + const __half2* __restrict__ vec,
28 #endif
29 - const int* __restrict__ mat,
29 + const int* __restrict__ mat,
30 #ifndef USE_ROCM
31 - half2* __restrict__ mul,
31 + half2* __restrict__ mul,
32 #else
33 - float2* __restrict__ mul,
33 + float2* __restrict__ mul,
34 #endif
35 - const __half* __restrict__ lookup_table,
36 - int height,
37 - int width,
38 - int batch,
39 - int vec_height
40 -) {
35 + const __half* __restrict__ lookup_table, int height, int width, int batch,
36 + int vec_height) {
37
38 const int blockwidth2 = BLOCKWIDTH / 2;
39
40 int row = BLOCKHEIGHT4 * blockIdx.x;
45 - int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
41 + int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
42
43 #ifndef USE_ROCM
44 __shared__ half2 blockvec[blockwidth2];
@@ -73,14 +69,16 @@ __global__ void NUQ4MatMulKernel(
69 unsigned int tmp1;
70 unsigned int lut_index1, lut_index2;
71
76 - for (int b = 0; b < batch; ++b){
72 + for (int b = 0; b < batch; ++b) {
73 i = width * row + col;
74 res = __int2half_rd(0);
75 k = 0;
76
77 __syncthreads();
78 if (threadIdx.x < blockwidth2)
83 - blockvec[threadIdx.x] = vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 + threadIdx.x];
79 + blockvec[threadIdx.x] =
80 + vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 +
81 + threadIdx.x];
82 __syncthreads();
83
84 while (k < blockwidth2) {
@@ -143,7 +141,8 @@ __global__ void NUQ4MatMulKernel(
141 #ifndef USE_ROCM
142 res = __hadd(__hadd(res2.x, res2.y), res);
143 #else
146 - res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)), res);
144 + res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)),
145 + res);
146 #endif
147
148 i += width;
@@ -179,46 +178,38 @@ __global__ void NUQ4MatMulKernel(
178 }
179 }
180
182 -} // namespace squeezellm
183 -} // namespace vllm
181 +} // namespace squeezellm
182 +} // namespace vllm
183
184 // 4-bit matvec kernel (LUT-based)
186 -void squeezellm_gemm(
187 - torch::Tensor vec,
188 - torch::Tensor mat,
189 - torch::Tensor mul,
190 - torch::Tensor lookup_table
191 -) {
185 +void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
186 + torch::Tensor lookup_table) {
187 int height = mat.size(0);
188 int width = mat.size(1);
189
190 int batch = vec.size(0);
191 int vec_height = vec.size(1);
192
198 - dim3 blocks(
199 - (height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
200 - (width + BLOCKWIDTH - 1) / BLOCKWIDTH
201 - );
193 + dim3 blocks((height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
194 + (width + BLOCKWIDTH - 1) / BLOCKWIDTH);
195 dim3 threads(BLOCKWIDTH);
196
197 const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
198 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
199 vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
200 #ifndef USE_ROCM
208 - (half2*) vec.data<at::Half>(),
201 + (half2*)vec.data<at::Half>(),
202 #else
210 - (__half2*) vec.data_ptr<at::Half>(),
203 + (__half2*)vec.data_ptr<at::Half>(),
204 #endif
212 - mat.data_ptr<int>(),
205 + mat.data_ptr<int>(),
206 #ifndef USE_ROCM
214 - (half2*) mul.data<at::Half>(),
215 - (__half*) lookup_table.data<at::Half>(),
207 + (half2*)mul.data<at::Half>(), (__half*)lookup_table.data<at::Half>(),
208 #else
217 - (float2*) mul.data_ptr<float>(),
218 - (__half*) lookup_table.data_ptr<at::Half>(),
209 + (float2*)mul.data_ptr<float>(),
210 + (__half*)lookup_table.data_ptr<at::Half>(),
211 #endif
220 - height, width, batch, vec_height
221 - );
212 + height, width, batch, vec_height);
213 }
214
215 #undef BLOCKWIDTH
csrc/reduction_utils.cuh
+11 -9
@@ -1,5 +1,6 @@
1 /*
2 - * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
4 * Copyright (c) 2023, The vLLM team.
5 * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 *
@@ -20,12 +21,12 @@
21 #include "cuda_compat.h"
22
23 namespace vllm {
23 -template<typename T, int numLanes = WARP_SIZE>
24 +template <typename T, int numLanes = WARP_SIZE>
25 __inline__ __device__ T warpReduceSum(T val) {
26 static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
27 "numLanes is not a positive power of 2!");
28 static_assert(numLanes <= WARP_SIZE);
28 - #pragma unroll
29 +#pragma unroll
30 for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
31 val += VLLM_SHFL_XOR_SYNC(val, mask);
32 return val;
@@ -38,22 +39,23 @@ static constexpr int _nextPow2(unsigned int num) {
39 }
40
41 /* Calculate the sum of all elements in a block */
41 -template<typename T, int maxBlockSize = 1024>
42 +template <typename T, int maxBlockSize = 1024>
43 __inline__ __device__ T blockReduceSum(T val) {
44 static_assert(maxBlockSize <= 1024);
45 if constexpr (maxBlockSize > WARP_SIZE) {
46 val = warpReduceSum<T>(val);
46 - // Calculates max number of lanes that need to participate in the last warpReduce
47 + // Calculates max number of lanes that need to participate in the last
48 + // warpReduce
49 constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
50 static __shared__ T shared[maxActiveLanes];
51 int lane = threadIdx.x % WARP_SIZE;
52 int wid = threadIdx.x / WARP_SIZE;
51 - if (lane == 0)
52 - shared[wid] = val;
53 + if (lane == 0) shared[wid] = val;
54
55 __syncthreads();
56
56 - val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane] : (T)(0.0f);
57 + val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane]
58 + : (T)(0.0f);
59 val = warpReduceSum<T, _nextPow2(maxActiveLanes)>(val);
60 } else {
61 // A single warpReduce is equal to blockReduce
@@ -62,4 +64,4 @@ __inline__ __device__ T blockReduceSum(T val) {
64 return val;
65 }
66
65 -} // namespace vllm
67 +} // namespace vllm
format.sh
+56 -1
@@ -26,6 +26,7 @@ RUFF_VERSION=$(ruff --version | awk '{print $2}')
26 MYPY_VERSION=$(mypy --version | awk '{print $2}')
27 CODESPELL_VERSION=$(codespell --version)
28 ISORT_VERSION=$(isort --vn)
29 +CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
30
31 # # params: tool name, tool version, required version
32 tool_version_check() {
@@ -40,6 +41,7 @@ tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt |
41 tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
42 tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-dev.txt | cut -d'=' -f3)"
43 tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
44 +tool_version_check "clang-format" "$CLANGFORMAT_VERSION" "$(grep clang-format requirements-dev.txt | cut -d'=' -f3)"
45
46 YAPF_FLAGS=(
47 '--recursive'
@@ -179,7 +181,6 @@ lint_changed() {
181 }
182
183 # Run Ruff
182 -echo 'vLLM ruff:'
184 ### This flag lints individual files. --files *must* be the first command line
185 ### arg to use this option.
186 if [[ "$1" == '--files' ]]; then
@@ -192,6 +193,7 @@ else
193 # Format only the files that changed in last commit.
194 lint_changed
195 fi
196 +echo 'vLLM ruff: Done'
197
198 # check spelling of specified files
199 isort_check() {
@@ -233,6 +235,59 @@ else
235 fi
236 echo 'vLLM isort: Done'
237
238 +# Clang-format section
239 +# Exclude some files for formatting because they are vendored
240 +# NOTE: Keep up to date with .github/workflows/clang-format.yml
241 +CLANG_FORMAT_EXCLUDES=(
242 + 'csrc/moe/topk_softmax_kernels.cu'
243 + 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
244 + 'csrc/punica/bgmv/bgmv_config.h'
245 + 'csrc/punica/bgmv/bgmv_impl.cuh'
246 + 'csrc/punica/bgmv/vec_dtypes.cuh'
247 + 'csrc/punica/punica_ops.cu'
248 + 'csrc/punica/type_convert.h'
249 +)
250 +
251 +# Format specified files with clang-format
252 +clang_format() {
253 + clang-format -i "$@"
254 +}
255 +
256 +# Format files that differ from main branch with clang-format.
257 +clang_format_changed() {
258 + # The `if` guard ensures that the list of filenames is not empty, which
259 + # could cause clang-format to receive 0 positional arguments, making it hang
260 + # waiting for STDIN.
261 + #
262 + # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
263 + # exist on both branches.
264 + MERGEBASE="$(git merge-base origin/main HEAD)"
265 +
266 + # Get the list of changed files, excluding the specified ones
267 + changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}"))
268 + if [ -n "$changed_files" ]; then
269 + echo "$changed_files" | xargs -P 5 clang-format -i
270 + fi
271 +}
272 +
273 +# Format all files with clang-format
274 +clang_format_all() {
275 + find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
276 + | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") \
277 + | xargs clang-format -i
278 +}
279 +
280 +# Run clang-format
281 +if [[ "$1" == '--files' ]]; then
282 + clang_format "${@:2}"
283 +elif [[ "$1" == '--all' ]]; then
284 + clang_format_all
285 +else
286 + clang_format_changed
287 +fi
288 +echo 'vLLM clang-format: Done'
289 +
290 +
291 if ! git diff --quiet &>/dev/null; then
292 echo 'Reformatted files. Please review and stage the changes.'
293 echo 'Changes not staged for commit:'
requirements-dev.txt
+1
@@ -5,6 +5,7 @@ tomli==2.0.1
5 ruff==0.1.5
6 codespell==2.2.6
7 isort==5.13.2
8 +clang-format==18.1.5
9
10 # type checking
11 mypy==1.9.0