A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Build] Enforce style for C++ and CUDA code with `clang-format` (#4722)
Michael Goin committed
May 22, 2024 at 03:18 UTC
5f6d10c14c17122e6d711a4829ee0ca672e07f6f
64 files changed
+6571
-6963
.clang-format
new
+26
@@ -0,0 +1,26 @@
1
+BasedOnStyle: Google
2
+UseTab: Never
3
+IndentWidth: 2
4
+ColumnLimit: 80
5
+
6
+# Force pointers to the type for C++.
7
+DerivePointerAlignment: false
8
+PointerAlignment: Left
9
+
10
+# Reordering #include statements can (and currently will) introduce errors
11
+SortIncludes: false
12
+
13
+# Style choices
14
+AlignConsecutiveAssignments: false
15
+AlignConsecutiveDeclarations: false
16
+IndentPPDirectives: BeforeHash
17
+
18
+IncludeCategories:
19
+ - Regex: '^<'
20
+ Priority: 4
21
+ - Regex: '^"(llvm|llvm-c|clang|clang-c|mlir|mlir-c)/'
22
+ Priority: 3
23
+ - Regex: '^"(qoda|\.\.)/'
24
+ Priority: 2
25
+ - Regex: '.*'
26
+ Priority: 1
.github/workflows/clang-format.yml
new
+42
@@ -0,0 +1,42 @@
1
+name: clang-format
2
+
3
+on:
4
+ # Trigger the workflow on push or pull request,
5
+ # but only for the main branch
6
+ push:
7
+ branches:
8
+ - main
9
+ pull_request:
10
+ branches:
11
+ - main
12
+
13
+jobs:
14
+ clang-format:
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ matrix:
18
+ python-version: ["3.11"]
19
+ steps:
20
+ - uses: actions/checkout@v2
21
+ - name: Set up Python ${{ matrix.python-version }}
22
+ uses: actions/setup-python@v2
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+ - name: Install dependencies
26
+ run: |
27
+ python -m pip install --upgrade pip
28
+ pip install clang-format==18.1.5
29
+ - name: Running clang-format
30
+ run: |
31
+ EXCLUDES=(
32
+ 'csrc/moe/topk_softmax_kernels.cu'
33
+ 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
34
+ 'csrc/punica/bgmv/bgmv_config.h'
35
+ 'csrc/punica/bgmv/bgmv_impl.cuh'
36
+ 'csrc/punica/bgmv/vec_dtypes.cuh'
37
+ 'csrc/punica/punica_ops.cu'
38
+ 'csrc/punica/type_convert.h'
39
+ )
40
+ find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
41
+ | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
42
+ | xargs clang-format --dry-run --Werror
\ No newline at end of file
csrc/activation_kernels.cu
+64
-75
@@ -10,11 +10,11 @@
10
namespace vllm {
11
12
// Activation and gating kernel template.
13
-template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
13
+template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
14
__global__ void act_and_mul_kernel(
15
- scalar_t* __restrict__ out, // [..., d]
16
- const scalar_t* __restrict__ input, // [..., 2, d]
17
- const int d) {
15
+ scalar_t* __restrict__ out, // [..., d]
16
+ const scalar_t* __restrict__ input, // [..., 2, d]
17
+ const int d) {
18
const int64_t token_idx = blockIdx.x;
19
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
20
const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
@@ -23,72 +23,66 @@ __global__ void act_and_mul_kernel(
23
}
24
}
25
26
-template<typename T>
26
+template <typename T>
27
__device__ __forceinline__ T silu_kernel(const T& x) {
28
// x * sigmoid(x)
29
- return (T) (((float) x) / (1.0f + expf((float) -x)));
29
+ return (T)(((float)x) / (1.0f + expf((float)-x)));
30
}
31
32
-template<typename T>
32
+template <typename T>
33
__device__ __forceinline__ T gelu_kernel(const T& x) {
34
// Equivalent to PyTorch GELU with 'none' approximation.
35
// Refer to:
36
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
37
- const float f = (float) x;
37
+ const float f = (float)x;
38
constexpr float ALPHA = M_SQRT1_2;
39
- return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
39
+ return (T)(f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40
}
41
42
-template<typename T>
42
+template <typename T>
43
__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
44
// Equivalent to PyTorch GELU with 'tanh' approximation.
45
// Refer to:
46
// https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
47
- const float f = (float) x;
47
+ const float f = (float)x;
48
constexpr float BETA = M_SQRT2 * M_2_SQRTPI * 0.5f;
49
constexpr float KAPPA = 0.044715;
50
float x_cube = f * f * f;
51
float inner = BETA * (f + KAPPA * x_cube);
52
- return (T) (0.5f * f * (1.0f + ::tanhf(inner)));
52
+ return (T)(0.5f * f * (1.0f + ::tanhf(inner)));
53
}
54
55
-} // namespace vllm
55
+} // namespace vllm
56
57
// Launch activation and gating kernel.
58
-#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
59
- int d = input.size(-1) / 2; \
60
- int64_t num_tokens = input.numel() / input.size(-1); \
61
- dim3 grid(num_tokens); \
62
- dim3 block(std::min(d, 1024)); \
63
- const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
64
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
65
- VLLM_DISPATCH_FLOATING_TYPES( \
66
- input.scalar_type(), \
67
- "act_and_mul_kernel", \
68
- [&] { \
69
- vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
70
- out.data_ptr<scalar_t>(), \
71
- input.data_ptr<scalar_t>(), \
72
- d); \
73
- });
74
-
75
-void silu_and_mul(
76
- torch::Tensor& out, // [..., d]
77
- torch::Tensor& input) // [..., 2 * d]
58
+#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
59
+ int d = input.size(-1) / 2; \
60
+ int64_t num_tokens = input.numel() / input.size(-1); \
61
+ dim3 grid(num_tokens); \
62
+ dim3 block(std::min(d, 1024)); \
63
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
64
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
65
+ VLLM_DISPATCH_FLOATING_TYPES( \
66
+ input.scalar_type(), "act_and_mul_kernel", [&] { \
67
+ vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>> \
68
+ <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
69
+ input.data_ptr<scalar_t>(), d); \
70
+ });
71
+
72
+void silu_and_mul(torch::Tensor& out, // [..., d]
73
+ torch::Tensor& input) // [..., 2 * d]
74
{
75
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
76
}
77
82
-void gelu_and_mul(
83
- torch::Tensor& out, // [..., d]
84
- torch::Tensor& input) // [..., 2 * d]
78
+void gelu_and_mul(torch::Tensor& out, // [..., d]
79
+ torch::Tensor& input) // [..., 2 * d]
80
{
81
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
82
}
83
89
-void gelu_tanh_and_mul(
90
- torch::Tensor& out, // [..., d]
91
- torch::Tensor& input) // [..., 2 * d]
84
+void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
85
+ torch::Tensor& input) // [..., 2 * d]
86
{
87
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
88
}
@@ -96,11 +90,11 @@ void gelu_tanh_and_mul(
90
namespace vllm {
91
92
// Element-wise activation kernel template.
99
-template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
93
+template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
94
__global__ void activation_kernel(
101
- scalar_t* __restrict__ out, // [..., d]
102
- const scalar_t* __restrict__ input, // [..., d]
103
- const int d) {
95
+ scalar_t* __restrict__ out, // [..., d]
96
+ const scalar_t* __restrict__ input, // [..., d]
97
+ const int d) {
98
const int64_t token_idx = blockIdx.x;
99
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
100
const scalar_t x = VLLM_LDG(&input[token_idx * d + idx]);
@@ -108,54 +102,49 @@ __global__ void activation_kernel(
102
}
103
}
104
111
-} // namespace vllm
105
+} // namespace vllm
106
107
// Launch element-wise activation kernel.
114
-#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
115
- int d = input.size(-1); \
116
- int64_t num_tokens = input.numel() / d; \
117
- dim3 grid(num_tokens); \
118
- dim3 block(std::min(d, 1024)); \
119
- const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
120
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
121
- VLLM_DISPATCH_FLOATING_TYPES( \
122
- input.scalar_type(), \
123
- "activation_kernel", \
124
- [&] { \
125
- vllm::activation_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
126
- out.data_ptr<scalar_t>(), \
127
- input.data_ptr<scalar_t>(), \
128
- d); \
129
- });
108
+#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
109
+ int d = input.size(-1); \
110
+ int64_t num_tokens = input.numel() / d; \
111
+ dim3 grid(num_tokens); \
112
+ dim3 block(std::min(d, 1024)); \
113
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
114
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
115
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "activation_kernel", [&] { \
116
+ vllm::activation_kernel<scalar_t, KERNEL<scalar_t>> \
117
+ <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
118
+ input.data_ptr<scalar_t>(), d); \
119
+ });
120
121
namespace vllm {
122
133
-template<typename T>
123
+template <typename T>
124
__device__ __forceinline__ T gelu_new_kernel(const T& x) {
135
- const float x3 = (float) (x * x * x);
136
- const T t = (T) tanhf((T) (0.79788456f * (float) (x + (T) (0.044715f * x3))));
137
- return ((T) 0.5) * x * (((T) 1.0) + t);
125
+ const float x3 = (float)(x * x * x);
126
+ const T t = (T)tanhf((T)(0.79788456f * (float)(x + (T)(0.044715f * x3))));
127
+ return ((T)0.5) * x * (((T)1.0) + t);
128
}
129
140
-template<typename T>
130
+template <typename T>
131
__device__ __forceinline__ T gelu_fast_kernel(const T& x) {
142
- const float f = (float) x;
143
- const T t = (T) tanhf(((T) (f * 0.79788456f)) * (((T) 1.0) + (T) (0.044715f * f) * x));
144
- return ((T) 0.5) * x * (((T) 1.0) + t);
132
+ const float f = (float)x;
133
+ const T t =
134
+ (T)tanhf(((T)(f * 0.79788456f)) * (((T)1.0) + (T)(0.044715f * f) * x));
135
+ return ((T)0.5) * x * (((T)1.0) + t);
136
}
137
147
-} // namespace vllm
138
+} // namespace vllm
139
149
-void gelu_new(
150
- torch::Tensor& out, // [..., d]
151
- torch::Tensor& input) // [..., d]
140
+void gelu_new(torch::Tensor& out, // [..., d]
141
+ torch::Tensor& input) // [..., d]
142
{
143
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_new_kernel);
144
}
145
156
-void gelu_fast(
157
- torch::Tensor& out, // [..., d]
158
- torch::Tensor& input) // [..., d]
146
+void gelu_fast(torch::Tensor& out, // [..., d]
147
+ torch::Tensor& input) // [..., d]
148
{
149
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
150
}
csrc/attention/attention_generic.cuh
+10
-9
@@ -1,5 +1,6 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
4
* Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
@@ -22,31 +23,31 @@
23
namespace vllm {
24
25
// A vector type to store Q, K, V elements.
25
-template<typename T, int VEC_SIZE>
26
+template <typename T, int VEC_SIZE>
27
struct Vec {};
28
29
// A vector type to store FP32 accumulators.
29
-template<typename T>
30
+template <typename T>
31
struct FloatVec {};
32
33
// Template vector operations.
33
-template<typename Acc, typename A, typename B>
34
+template <typename Acc, typename A, typename B>
35
inline __device__ Acc mul(A a, B b);
36
36
-template<typename T>
37
+template <typename T>
38
inline __device__ float sum(T v);
39
39
-template<typename T>
40
+template <typename T>
41
inline __device__ float dot(T a, T b) {
42
return sum(mul<T, T, T>(a, b));
43
}
44
44
-template<typename A, typename T>
45
+template <typename A, typename T>
46
inline __device__ float dot(T a, T b) {
47
return sum(mul<A, T, T>(a, b));
48
}
49
49
-template<typename T>
50
+template <typename T>
51
inline __device__ void zero(T& dst) {
52
constexpr int WORDS = sizeof(T) / 4;
53
union {
@@ -61,4 +62,4 @@ inline __device__ void zero(T& dst) {
62
dst = tmp.raw;
63
}
64
64
-} // namespace vllm
65
+} // namespace vllm
csrc/attention/attention_kernels.cu
+299
-337
@@ -1,5 +1,6 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
* Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
@@ -27,15 +28,15 @@
28
#ifdef USE_ROCM
29
#include <hip/hip_bf16.h>
30
#include "../quantization/fp8/amd/quant_utils.cuh"
30
- typedef __hip_bfloat16 __nv_bfloat16;
31
+typedef __hip_bfloat16 __nv_bfloat16;
32
#else
33
#include "../quantization/fp8/nvidia/quant_utils.cuh"
34
#endif
35
36
#ifndef USE_ROCM
36
-#define WARP_SIZE 32
37
+ #define WARP_SIZE 32
38
#else
38
-#define WARP_SIZE warpSize
39
+ #define WARP_SIZE warpSize
40
#endif
41
42
#define MAX(a, b) ((a) > (b) ? (a) : (b))
@@ -45,7 +46,7 @@
46
namespace vllm {
47
48
// Utility function for attention softmax.
48
-template<int NUM_WARPS>
49
+template <int NUM_WARPS>
50
inline __device__ float block_sum(float* red_smem, float sum) {
51
// Decompose the thread index into warp / lane.
52
int warp = threadIdx.x / WARP_SIZE;
@@ -82,31 +83,28 @@ inline __device__ float block_sum(float* red_smem, float sum) {
83
84
// TODO(woosuk): Merge the last two dimensions of the grid.
85
// Grid: (num_heads, num_seqs, max_num_partitions).
85
-template<
86
- typename scalar_t,
87
- typename cache_t,
88
- int HEAD_SIZE,
89
- int BLOCK_SIZE,
90
- int NUM_THREADS,
91
- vllm::Fp8KVCacheDataType KV_DTYPE,
92
- int PARTITION_SIZE = 0> // Zero means no partitioning.
86
+template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
87
+ int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
88
+ int PARTITION_SIZE = 0> // Zero means no partitioning.
89
__device__ void paged_attention_kernel(
94
- float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
95
- float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
96
- scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
97
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
98
- const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
99
- const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
100
- const int num_kv_heads, // [num_heads]
101
- const float scale,
102
- const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
103
- const int* __restrict__ seq_lens, // [num_seqs]
104
- const int max_num_blocks_per_seq,
105
- const float* __restrict__ alibi_slopes, // [num_heads]
106
- const int q_stride,
107
- const int kv_block_stride,
108
- const int kv_head_stride,
109
- const float kv_scale) {
90
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
91
+ float* __restrict__ max_logits, // [num_seqs, num_heads,
92
+ // max_num_partitions]
93
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
94
+ // head_size]
95
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
96
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
97
+ // head_size/x, block_size, x]
98
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
99
+ // head_size, block_size]
100
+ const int num_kv_heads, // [num_heads]
101
+ const float scale,
102
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
103
+ const int* __restrict__ seq_lens, // [num_seqs]
104
+ const int max_num_blocks_per_seq,
105
+ const float* __restrict__ alibi_slopes, // [num_heads]
106
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
107
+ const float kv_scale) {
108
const int seq_idx = blockIdx.y;
109
const int partition_idx = blockIdx.z;
110
const int max_num_partitions = gridDim.z;
@@ -118,22 +116,29 @@ __device__ void paged_attention_kernel(
116
}
117
118
const int num_seq_blocks = DIVIDE_ROUND_UP(seq_len, BLOCK_SIZE);
121
- const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
119
+ const int num_blocks_per_partition =
120
+ USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
121
122
// [start_block_idx, end_block_idx) is the range of blocks to process.
124
- const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
125
- const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
123
+ const int start_block_idx =
124
+ USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
125
+ const int end_block_idx =
126
+ MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
127
const int num_blocks = end_block_idx - start_block_idx;
128
129
// [start_token_idx, end_token_idx) is the range of tokens to process.
130
const int start_token_idx = start_block_idx * BLOCK_SIZE;
130
- const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
131
+ const int end_token_idx =
132
+ MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
133
const int num_tokens = end_token_idx - start_token_idx;
134
135
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
134
- constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
136
+ constexpr int NUM_THREAD_GROUPS =
137
+ NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE
138
+ // divides NUM_THREADS
139
assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
136
- constexpr int NUM_TOKENS_PER_THREAD_GROUP = DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
140
+ constexpr int NUM_TOKENS_PER_THREAD_GROUP =
141
+ DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
142
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
143
const int thread_idx = threadIdx.x;
144
const int warp_idx = thread_idx / WARP_SIZE;
@@ -143,13 +148,14 @@ __device__ void paged_attention_kernel(
148
const int num_heads = gridDim.x;
149
const int num_queries_per_kv = num_heads / num_kv_heads;
150
const int kv_head_idx = head_idx / num_queries_per_kv;
146
- const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
151
+ const float alibi_slope =
152
+ alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
153
154
// A vector type to store a part of a key or a query.
149
- // The vector size is configured in such a way that the threads in a thread group
150
- // fetch or compute 16 bytes at a time.
151
- // For example, if the size of a thread group is 4 and the data type is half,
152
- // then the vector size is 16 / (4 * sizeof(half)) == 2.
155
+ // The vector size is configured in such a way that the threads in a thread
156
+ // group fetch or compute 16 bytes at a time. For example, if the size of a
157
+ // thread group is 4 and the data type is half, then the vector size is 16 /
158
+ // (4 * sizeof(half)) == 2.
159
constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
160
using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
161
using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
@@ -163,18 +169,21 @@ __device__ void paged_attention_kernel(
169
170
// Load the query to registers.
171
// Each thread in a thread group has a different part of the query.
166
- // For example, if the the thread group size is 4, then the first thread in the group
167
- // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
168
- // th vectors of the query, and so on.
169
- // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
172
+ // For example, if the the thread group size is 4, then the first thread in
173
+ // the group has 0, 4, 8, ... th vectors of the query, and the second thread
174
+ // has 1, 5, 9, ... th vectors of the query, and so on. NOTE(woosuk): Because
175
+ // q is split from a qkv tensor, it may not be contiguous.
176
const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
177
__shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
178
#pragma unroll
173
- for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) {
179
+ for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD;
180
+ i += NUM_THREAD_GROUPS) {
181
const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
175
- q_vecs[thread_group_offset][i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
182
+ q_vecs[thread_group_offset][i] =
183
+ *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
184
}
177
- __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a memory wall right before we use q_vecs
185
+ __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a
186
+ // memory wall right before we use q_vecs
187
188
// Memory planning.
189
extern __shared__ char shared_mem[];
@@ -193,44 +202,50 @@ __device__ void paged_attention_kernel(
202
// Each thread group in a warp fetches a key from the block, and computes
203
// dot product with the query.
204
const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
196
- for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
197
- // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
198
- // because int32 can lead to overflow when this variable is multiplied by large numbers
199
- // (e.g., kv_block_stride).
200
- const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
205
+ for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
206
+ block_idx += NUM_WARPS) {
207
+ // NOTE(woosuk): The block number is stored in int32. However, we cast it to
208
+ // int64 because int32 can lead to overflow when this variable is multiplied
209
+ // by large numbers (e.g., kv_block_stride).
210
+ const int64_t physical_block_number =
211
+ static_cast<int64_t>(block_table[block_idx]);
212
213
// Load a key to registers.
214
// Each thread in a thread group has a different part of the key.
204
- // For example, if the the thread group size is 4, then the first thread in the group
205
- // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
206
- // vectors of the key, and so on.
215
+ // For example, if the the thread group size is 4, then the first thread in
216
+ // the group has 0, 4, 8, ... th vectors of the key, and the second thread
217
+ // has 1, 5, 9, ... th vectors of the key, and so on.
218
for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
208
- const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
219
+ const int physical_block_offset =
220
+ (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
221
const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
222
K_vec k_vecs[NUM_VECS_PER_THREAD];
223
224
#pragma unroll
225
for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
214
- const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride
215
- + kv_head_idx * kv_head_stride
216
- + physical_block_offset * x;
226
+ const cache_t* k_ptr =
227
+ k_cache + physical_block_number * kv_block_stride +
228
+ kv_head_idx * kv_head_stride + physical_block_offset * x;
229
const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
230
const int offset1 = (vec_idx * VEC_SIZE) / x;
231
const int offset2 = (vec_idx * VEC_SIZE) % x;
232
233
if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
222
- k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
234
+ k_vecs[j] = *reinterpret_cast<const K_vec*>(
235
+ k_ptr + offset1 * BLOCK_SIZE * x + offset2);
236
} else {
237
// Vector conversion from Quant_vec to K_vec.
238
Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
226
- k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227
- k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(k_vec_quant, kv_scale);
239
+ k_ptr + offset1 * BLOCK_SIZE * x + offset2);
240
+ k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
241
+ k_vec_quant, kv_scale);
242
}
243
}
244
245
// Compute dot product.
246
// This includes a reduction across the threads in the same thread group.
233
- float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
247
+ float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(
248
+ q_vecs[thread_group_offset], k_vecs);
249
// Add the ALiBi bias if slopes are given.
250
qk += (alibi_slope != 0) ? alibi_slope * (token_idx - seq_len + 1) : 0;
251
@@ -285,13 +300,12 @@ __device__ void paged_attention_kernel(
300
301
// If partitioning is enabled, store the max logit and exp_sum.
302
if (USE_PARTITIONING && thread_idx == 0) {
288
- float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
289
- + head_idx * max_num_partitions
290
- + partition_idx;
303
+ float* max_logits_ptr = max_logits +
304
+ seq_idx * num_heads * max_num_partitions +
305
+ head_idx * max_num_partitions + partition_idx;
306
*max_logits_ptr = qk_max;
292
- float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
293
- + head_idx * max_num_partitions
294
- + partition_idx;
307
+ float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions +
308
+ head_idx * max_num_partitions + partition_idx;
309
*exp_sums_ptr = exp_sum;
310
}
311
@@ -304,7 +318,8 @@ __device__ void paged_attention_kernel(
318
319
constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
320
constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
307
- constexpr int NUM_ROWS_PER_THREAD = DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
321
+ constexpr int NUM_ROWS_PER_THREAD =
322
+ DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
323
324
// NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
325
float accs[NUM_ROWS_PER_THREAD];
@@ -315,18 +330,21 @@ __device__ void paged_attention_kernel(
330
331
scalar_t zero_value;
332
zero(zero_value);
318
- for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
319
- // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
320
- // because int32 can lead to overflow when this variable is multiplied by large numbers
321
- // (e.g., kv_block_stride).
322
- const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
333
+ for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
334
+ block_idx += NUM_WARPS) {
335
+ // NOTE(woosuk): The block number is stored in int32. However, we cast it to
336
+ // int64 because int32 can lead to overflow when this variable is multiplied
337
+ // by large numbers (e.g., kv_block_stride).
338
+ const int64_t physical_block_number =
339
+ static_cast<int64_t>(block_table[block_idx]);
340
const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
341
const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
342
L_vec logits_vec;
326
- from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
343
+ from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx -
344
+ start_token_idx));
345
328
- const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride
329
- + kv_head_idx * kv_head_stride;
346
+ const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride +
347
+ kv_head_idx * kv_head_stride;
348
#pragma unroll
349
for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
350
const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -337,14 +355,17 @@ __device__ void paged_attention_kernel(
355
if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
356
v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
357
} else {
340
- V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358
+ V_quant_vec v_quant_vec =
359
+ *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
360
// Vector conversion from V_quant_vec to V_vec.
342
- v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec, kv_scale);
361
+ v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
362
+ kv_scale);
363
}
364
if (block_idx == num_seq_blocks - 1) {
345
- // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
346
- // we should explicitly zero out the values since they may contain NaNs.
347
- // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
365
+ // NOTE(woosuk): When v_vec contains the tokens that are out of the
366
+ // context, we should explicitly zero out the values since they may
367
+ // contain NaNs. See
368
+ // https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
369
scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
370
#pragma unroll
371
for (int j = 0; j < V_VEC_SIZE; j++) {
@@ -367,8 +388,8 @@ __device__ void paged_attention_kernel(
388
accs[i] = acc;
389
}
390
370
- // NOTE(woosuk): A barrier is required because the shared memory space for logits
371
- // is reused for the output.
391
+ // NOTE(woosuk): A barrier is required because the shared memory space for
392
+ // logits is reused for the output.
393
__syncthreads();
394
395
// Perform reduction across warps.
@@ -405,9 +426,9 @@ __device__ void paged_attention_kernel(
426
427
// Write the final output.
428
if (warp_idx == 0) {
408
- scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
409
- + head_idx * max_num_partitions * HEAD_SIZE
410
- + partition_idx * HEAD_SIZE;
429
+ scalar_t* out_ptr =
430
+ out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
431
+ head_idx * max_num_partitions * HEAD_SIZE + partition_idx * HEAD_SIZE;
432
#pragma unroll
433
for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
434
const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -419,79 +440,75 @@ __device__ void paged_attention_kernel(
440
}
441
442
// Grid: (num_heads, num_seqs, 1).
422
-template<
423
- typename scalar_t,
424
- typename cache_t,
425
- int HEAD_SIZE,
426
- int BLOCK_SIZE,
427
- int NUM_THREADS,
428
- vllm::Fp8KVCacheDataType KV_DTYPE>
443
+template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
444
+ int NUM_THREADS,
445
+ vllm::Fp8KVCacheDataType KV_DTYPE>
446
__global__ void paged_attention_v1_kernel(
430
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
431
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
432
- const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
433
- const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
434
- const int num_kv_heads, // [num_heads]
435
- const float scale,
436
- const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
437
- const int* __restrict__ seq_lens, // [num_seqs]
438
- const int max_num_blocks_per_seq,
439
- const float* __restrict__ alibi_slopes, // [num_heads]
440
- const int q_stride,
441
- const int kv_block_stride,
442
- const int kv_head_stride,
443
- const float kv_scale) {
444
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>(
445
- /* exp_sums */ nullptr, /* max_logits */ nullptr,
446
- out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
447
- max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
447
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
448
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
449
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
450
+ // head_size/x, block_size, x]
451
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
452
+ // head_size, block_size]
453
+ const int num_kv_heads, // [num_heads]
454
+ const float scale,
455
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
456
+ const int* __restrict__ seq_lens, // [num_seqs]
457
+ const int max_num_blocks_per_seq,
458
+ const float* __restrict__ alibi_slopes, // [num_heads]
459
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
460
+ const float kv_scale) {
461
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
462
+ KV_DTYPE>(
463
+ /* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
464
+ v_cache, num_kv_heads, scale, block_tables, seq_lens,
465
+ max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
466
+ kv_head_stride, kv_scale);
467
}
468
469
// Grid: (num_heads, num_seqs, max_num_partitions).
451
-template<
452
- typename scalar_t,
453
- typename cache_t,
454
- int HEAD_SIZE,
455
- int BLOCK_SIZE,
456
- int NUM_THREADS,
457
- vllm::Fp8KVCacheDataType KV_DTYPE,
458
- int PARTITION_SIZE>
470
+template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
471
+ int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
472
+ int PARTITION_SIZE>
473
__global__ void paged_attention_v2_kernel(
460
- float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
461
- float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
462
- scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
463
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
464
- const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
465
- const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
466
- const int num_kv_heads, // [num_heads]
467
- const float scale,
468
- const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
469
- const int* __restrict__ seq_lens, // [num_seqs]
470
- const int max_num_blocks_per_seq,
471
- const float* __restrict__ alibi_slopes, // [num_heads]
472
- const int q_stride,
473
- const int kv_block_stride,
474
- const int kv_head_stride,
475
- const float kv_scale) {
476
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE, PARTITION_SIZE>(
477
- exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
478
- block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
479
- q_stride, kv_block_stride, kv_head_stride, kv_scale);
474
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
475
+ float* __restrict__ max_logits, // [num_seqs, num_heads,
476
+ // max_num_partitions]
477
+ scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
478
+ // max_num_partitions, head_size]
479
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
480
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
481
+ // head_size/x, block_size, x]
482
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
483
+ // head_size, block_size]
484
+ const int num_kv_heads, // [num_heads]
485
+ const float scale,
486
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
487
+ const int* __restrict__ seq_lens, // [num_seqs]
488
+ const int max_num_blocks_per_seq,
489
+ const float* __restrict__ alibi_slopes, // [num_heads]
490
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
491
+ const float kv_scale) {
492
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
493
+ KV_DTYPE, PARTITION_SIZE>(
494
+ exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
495
+ block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
496
+ kv_block_stride, kv_head_stride, kv_scale);
497
}
498
499
// Grid: (num_heads, num_seqs).
483
-template<
484
- typename scalar_t,
485
- int HEAD_SIZE,
486
- int NUM_THREADS,
487
- int PARTITION_SIZE>
500
+template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
501
+ int PARTITION_SIZE>
502
__global__ void paged_attention_v2_reduce_kernel(
489
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
490
- const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
491
- const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
492
- const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
493
- const int* __restrict__ seq_lens, // [num_seqs]
494
- const int max_num_partitions) {
503
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
504
+ const float* __restrict__ exp_sums, // [num_seqs, num_heads,
505
+ // max_num_partitions]
506
+ const float* __restrict__ max_logits, // [num_seqs, num_heads,
507
+ // max_num_partitions]
508
+ const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
509
+ // max_num_partitions, head_size]
510
+ const int* __restrict__ seq_lens, // [num_seqs]
511
+ const int max_num_partitions) {
512
const int num_heads = gridDim.x;
513
const int head_idx = blockIdx.x;
514
const int seq_idx = blockIdx.y;
@@ -499,9 +516,11 @@ __global__ void paged_attention_v2_reduce_kernel(
516
const int num_partitions = DIVIDE_ROUND_UP(seq_len, PARTITION_SIZE);
517
if (num_partitions == 1) {
518
// No need to reduce. Only copy tmp_out to out.
502
- scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
503
- const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
504
- + head_idx * max_num_partitions * HEAD_SIZE;
519
+ scalar_t* out_ptr =
520
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
521
+ const scalar_t* tmp_out_ptr =
522
+ tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
523
+ head_idx * max_num_partitions * HEAD_SIZE;
524
for (int i = threadIdx.x; i < HEAD_SIZE; i += blockDim.x) {
525
out_ptr[i] = tmp_out_ptr[i];
526
}
@@ -520,8 +539,9 @@ __global__ void paged_attention_v2_reduce_kernel(
539
540
// Load max logits to shared memory.
541
float* shared_max_logits = reinterpret_cast<float*>(shared_mem);
523
- const float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
524
- + head_idx * max_num_partitions;
542
+ const float* max_logits_ptr = max_logits +
543
+ seq_idx * num_heads * max_num_partitions +
544
+ head_idx * max_num_partitions;
545
float max_logit = -FLT_MAX;
546
for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
547
const float l = max_logits_ptr[i];
@@ -550,9 +570,11 @@ __global__ void paged_attention_v2_reduce_kernel(
570
max_logit = VLLM_SHFL_SYNC(max_logit, 0);
571
572
// Load rescaled exp sums to shared memory.
553
- float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
554
- const float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
555
- + head_idx * max_num_partitions;
573
+ float* shared_exp_sums =
574
+ reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
575
+ const float* exp_sums_ptr = exp_sums +
576
+ seq_idx * num_heads * max_num_partitions +
577
+ head_idx * max_num_partitions;
578
float global_exp_sum = 0.0f;
579
for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
580
float l = shared_max_logits[i];
@@ -565,61 +587,45 @@ __global__ void paged_attention_v2_reduce_kernel(
587
const float inv_global_exp_sum = __fdividef(1.0f, global_exp_sum + 1e-6f);
588
589
// Aggregate tmp_out to out.
568
- const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
569
- + head_idx * max_num_partitions * HEAD_SIZE;
570
- scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
590
+ const scalar_t* tmp_out_ptr =
591
+ tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
592
+ head_idx * max_num_partitions * HEAD_SIZE;
593
+ scalar_t* out_ptr =
594
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
595
#pragma unroll
596
for (int i = threadIdx.x; i < HEAD_SIZE; i += NUM_THREADS) {
597
float acc = 0.0f;
598
for (int j = 0; j < num_partitions; ++j) {
575
- acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] * inv_global_exp_sum;
599
+ acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] *
600
+ inv_global_exp_sum;
601
}
602
from_float(out_ptr[i], acc);
603
}
604
}
605
581
-} // namespace vllm
582
-
583
-#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
584
- VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
585
- ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
586
- KV_DTYPE>), shared_mem_size); \
587
- vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
588
- KV_DTYPE><<<grid, block, shared_mem_size, stream>>>( \
589
- out_ptr, \
590
- query_ptr, \
591
- key_cache_ptr, \
592
- value_cache_ptr, \
593
- num_kv_heads, \
594
- scale, \
595
- block_tables_ptr, \
596
- seq_lens_ptr, \
597
- max_num_blocks_per_seq, \
598
- alibi_slopes_ptr, \
599
- q_stride, \
600
- kv_block_stride, \
601
- kv_head_stride, \
602
- kv_scale);
606
+} // namespace vllm
607
+
608
+#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
609
+ VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
610
+ ((void*)vllm::paged_attention_v1_kernel< \
611
+ T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>), \
612
+ shared_mem_size); \
613
+ vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
614
+ NUM_THREADS, KV_DTYPE> \
615
+ <<<grid, block, shared_mem_size, stream>>>( \
616
+ out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
617
+ scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
618
+ alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
619
+ kv_scale);
620
621
// TODO(woosuk): Tune NUM_THREADS.
605
-template<
606
- typename T,
607
- typename CACHE_T,
608
- int BLOCK_SIZE,
609
- vllm::Fp8KVCacheDataType KV_DTYPE,
610
- int NUM_THREADS = 128>
622
+template <typename T, typename CACHE_T, int BLOCK_SIZE,
623
+ vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128>
624
void paged_attention_v1_launcher(
612
- torch::Tensor& out,
613
- torch::Tensor& query,
614
- torch::Tensor& key_cache,
615
- torch::Tensor& value_cache,
616
- int num_kv_heads,
617
- float scale,
618
- torch::Tensor& block_tables,
619
- torch::Tensor& seq_lens,
620
- int max_seq_len,
621
- const c10::optional<torch::Tensor>& alibi_slopes,
622
- float kv_scale) {
625
+ torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
626
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
627
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
628
+ const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
629
int num_seqs = query.size(0);
630
int num_heads = query.size(1);
631
int head_size = query.size(2);
@@ -632,9 +638,10 @@ void paged_attention_v1_launcher(
638
assert(head_size % thread_group_size == 0);
639
640
// NOTE: alibi_slopes is optional.
635
- const float* alibi_slopes_ptr = alibi_slopes ?
636
- reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
637
- : nullptr;
641
+ const float* alibi_slopes_ptr =
642
+ alibi_slopes
643
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
644
+ : nullptr;
645
646
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
647
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
@@ -644,7 +651,8 @@ void paged_attention_v1_launcher(
651
int* seq_lens_ptr = seq_lens.data_ptr<int>();
652
653
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
647
- int padded_max_seq_len = DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
654
+ int padded_max_seq_len =
655
+ DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
656
int logits_size = padded_max_seq_len * sizeof(float);
657
int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
658
// Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
@@ -683,19 +691,10 @@ void paged_attention_v1_launcher(
691
}
692
}
693
686
-#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
687
- paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
688
- out, \
689
- query, \
690
- key_cache, \
691
- value_cache, \
692
- num_kv_heads, \
693
- scale, \
694
- block_tables, \
695
- seq_lens, \
696
- max_seq_len, \
697
- alibi_slopes, \
698
- kv_scale);
694
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
695
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
696
+ out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
697
+ seq_lens, max_seq_len, alibi_slopes, kv_scale);
698
699
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
700
// 1, 2, 4, 64, 128, 256.
@@ -716,74 +715,45 @@ void paged_attention_v1_launcher(
715
}
716
717
void paged_attention_v1(
719
- torch::Tensor& out, // [num_seqs, num_heads, head_size]
720
- torch::Tensor& query, // [num_seqs, num_heads, head_size]
721
- torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
722
- torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
723
- int num_kv_heads, // [num_heads]
724
- float scale,
725
- torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
726
- torch::Tensor& seq_lens, // [num_seqs]
727
- int block_size,
728
- int max_seq_len,
729
- const c10::optional<torch::Tensor>& alibi_slopes,
730
- const std::string& kv_cache_dtype,
731
- float kv_scale) {
732
-
733
- DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V1_LAUNCHER_BLOCK_SIZE)
734
-}
735
-
736
-#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
737
- vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
738
- KV_DTYPE, PARTITION_SIZE> \
739
- <<<grid, block, shared_mem_size, stream>>>( \
740
- exp_sums_ptr, \
741
- max_logits_ptr, \
742
- tmp_out_ptr, \
743
- query_ptr, \
744
- key_cache_ptr, \
745
- value_cache_ptr, \
746
- num_kv_heads, \
747
- scale, \
748
- block_tables_ptr, \
749
- seq_lens_ptr, \
750
- max_num_blocks_per_seq, \
751
- alibi_slopes_ptr, \
752
- q_stride, \
753
- kv_block_stride, \
754
- kv_head_stride, \
755
- kv_scale); \
756
- vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
757
- <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
758
- out_ptr, \
759
- exp_sums_ptr, \
760
- max_logits_ptr, \
761
- tmp_out_ptr, \
762
- seq_lens_ptr, \
763
- max_num_partitions);
764
-
765
-template<
766
- typename T,
767
- typename CACHE_T,
768
- int BLOCK_SIZE,
769
- vllm::Fp8KVCacheDataType KV_DTYPE,
770
- int NUM_THREADS = 128,
771
- int PARTITION_SIZE = 512>
718
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
719
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
720
+ torch::Tensor&
721
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
722
+ torch::Tensor&
723
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
724
+ int num_kv_heads, // [num_heads]
725
+ float scale,
726
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
727
+ torch::Tensor& seq_lens, // [num_seqs]
728
+ int block_size, int max_seq_len,
729
+ const c10::optional<torch::Tensor>& alibi_slopes,
730
+ const std::string& kv_cache_dtype, float kv_scale){
731
+
732
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
733
+ CALL_V1_LAUNCHER_BLOCK_SIZE)}
734
+#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
735
+ vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
736
+ NUM_THREADS, KV_DTYPE, PARTITION_SIZE> \
737
+ <<<grid, block, shared_mem_size, stream>>>( \
738
+ exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
739
+ value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
740
+ seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
741
+ kv_block_stride, kv_head_stride, kv_scale); \
742
+ vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
743
+ PARTITION_SIZE> \
744
+ <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
745
+ out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
746
+ max_num_partitions);
747
+
748
+template <typename T, typename CACHE_T, int BLOCK_SIZE,
749
+ vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128,
750
+ int PARTITION_SIZE = 512>
751
void paged_attention_v2_launcher(
773
- torch::Tensor& out,
774
- torch::Tensor& exp_sums,
775
- torch::Tensor& max_logits,
776
- torch::Tensor& tmp_out,
777
- torch::Tensor& query,
778
- torch::Tensor& key_cache,
779
- torch::Tensor& value_cache,
780
- int num_kv_heads,
781
- float scale,
782
- torch::Tensor& block_tables,
783
- torch::Tensor& seq_lens,
784
- int max_seq_len,
785
- const c10::optional<torch::Tensor>& alibi_slopes,
786
- float kv_scale) {
752
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
753
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
754
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
755
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
756
+ const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
757
int num_seqs = query.size(0);
758
int num_heads = query.size(1);
759
int head_size = query.size(2);
@@ -796,9 +766,10 @@ void paged_attention_v2_launcher(
766
assert(head_size % thread_group_size == 0);
767
768
// NOTE: alibi_slopes is optional.
799
- const float* alibi_slopes_ptr = alibi_slopes ?
800
- reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
801
- : nullptr;
769
+ const float* alibi_slopes_ptr =
770
+ alibi_slopes
771
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
772
+ : nullptr;
773
774
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
775
float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
@@ -853,59 +824,50 @@ void paged_attention_v2_launcher(
824
}
825
}
826
856
-#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
857
- paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
858
- out, \
859
- exp_sums, \
860
- max_logits, \
861
- tmp_out, \
862
- query, \
863
- key_cache, \
864
- value_cache, \
865
- num_kv_heads, \
866
- scale, \
867
- block_tables, \
868
- seq_lens, \
869
- max_seq_len, \
870
- alibi_slopes, \
871
- kv_scale);
827
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
828
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
829
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
830
+ num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
831
+ kv_scale);
832
833
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
834
// 1, 2, 4, 64, 128, 256.
875
-#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
876
- switch (block_size) { \
877
- case 8: \
878
- CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
879
- break; \
880
- case 16: \
881
- CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
882
- break; \
883
- case 32: \
884
- CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
885
- break; \
886
- default: \
887
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
888
- break; \
835
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
836
+ switch (block_size) { \
837
+ case 8: \
838
+ CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
839
+ break; \
840
+ case 16: \
841
+ CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
842
+ break; \
843
+ case 32: \
844
+ CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
845
+ break; \
846
+ default: \
847
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
848
+ break; \
849
}
850
851
void paged_attention_v2(
892
- torch::Tensor& out, // [num_seqs, num_heads, head_size]
893
- torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
894
- torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
895
- torch::Tensor& tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
896
- torch::Tensor& query, // [num_seqs, num_heads, head_size]
897
- torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
898
- torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
899
- int num_kv_heads, // [num_heads]
900
- float scale,
901
- torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
902
- torch::Tensor& seq_lens, // [num_seqs]
903
- int block_size,
904
- int max_seq_len,
905
- const c10::optional<torch::Tensor>& alibi_slopes,
906
- const std::string& kv_cache_dtype,
907
- float kv_scale) {
908
- DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V2_LAUNCHER_BLOCK_SIZE)
852
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
853
+ torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
854
+ torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
855
+ torch::Tensor&
856
+ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
857
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
858
+ torch::Tensor&
859
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
860
+ torch::Tensor&
861
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
862
+ int num_kv_heads, // [num_heads]
863
+ float scale,
864
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
865
+ torch::Tensor& seq_lens, // [num_seqs]
866
+ int block_size, int max_seq_len,
867
+ const c10::optional<torch::Tensor>& alibi_slopes,
868
+ const std::string& kv_cache_dtype, float kv_scale) {
869
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
870
+ CALL_V2_LAUNCHER_BLOCK_SIZE)
871
}
872
873
#undef WARP_SIZE
csrc/attention/attention_utils.cuh
+6
-5
@@ -1,5 +1,6 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
* Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
@@ -26,7 +27,7 @@
27
namespace vllm {
28
29
// Q*K^T operation.
29
-template<int THREAD_GROUP_SIZE, typename Vec, int N>
30
+template <int THREAD_GROUP_SIZE, typename Vec, int N>
31
inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
32
using A_vec = typename FloatVec<Vec>::Type;
33
// Compute the parallel products for Q*K^T (treat vector lanes separately).
@@ -45,12 +46,12 @@ inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
46
return qk;
47
}
48
48
-template<typename T, int THREAD_GROUP_SIZE>
49
+template <typename T, int THREAD_GROUP_SIZE>
50
struct Qk_dot {
50
- template<typename Vec, int N>
51
+ template <typename Vec, int N>
52
static inline __device__ float dot(const Vec (&q)[N], const Vec (&k)[N]) {
53
return qk_dot_<THREAD_GROUP_SIZE>(q, k);
54
}
55
};
56
56
-} // namespace vllm
57
+} // namespace vllm
csrc/attention/dtype_bfloat16.cuh
+39
-35
@@ -1,6 +1,8 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
- * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
+ * and
5
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6
* Copyright (c) 2023, The vLLM team.
7
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8
*
@@ -28,8 +30,8 @@
30
#include <hip/hip_bf16.h>
31
#include <hip/hip_fp16.h>
32
31
- typedef __hip_bfloat162 __nv_bfloat162;
32
- typedef __hip_bfloat16 __nv_bfloat16;
33
+typedef __hip_bfloat162 __nv_bfloat162;
34
+typedef __hip_bfloat16 __nv_bfloat16;
35
#endif
36
37
#include <stdint.h>
@@ -50,37 +52,37 @@ struct bf16_8_t {
52
};
53
54
// BF16 vector types for Q, K, V.
53
-template<>
55
+template <>
56
struct Vec<__nv_bfloat16, 1> {
57
using Type = __nv_bfloat16;
58
};
57
-template<>
59
+template <>
60
struct Vec<__nv_bfloat16, 2> {
61
using Type = __nv_bfloat162;
62
};
61
-template<>
63
+template <>
64
struct Vec<__nv_bfloat16, 4> {
65
using Type = bf16_4_t;
66
};
65
-template<>
67
+template <>
68
struct Vec<__nv_bfloat16, 8> {
69
using Type = bf16_8_t;
70
};
71
72
// FP32 accumulator vector types corresponding to Vec.
71
-template<>
73
+template <>
74
struct FloatVec<__nv_bfloat16> {
75
using Type = float;
76
};
75
-template<>
77
+template <>
78
struct FloatVec<__nv_bfloat162> {
79
using Type = float2;
80
};
79
-template<>
81
+template <>
82
struct FloatVec<bf16_4_t> {
83
using Type = Float4_;
84
};
83
-template<>
85
+template <>
86
struct FloatVec<bf16_8_t> {
87
using Type = Float8_;
88
};
@@ -108,9 +110,9 @@ inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
110
assert(false);
111
#else
112
#ifndef USE_ROCM
111
- return a + b;
113
+ return a + b;
114
#else
113
- return __hadd(a, b);
115
+ return __hadd(a, b);
116
#endif
117
#endif
118
}
@@ -161,7 +163,7 @@ inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
163
}
164
165
// Vector multiplication.
164
-template<>
166
+template <>
167
inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
168
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
169
assert(false);
@@ -170,7 +172,7 @@ inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
172
#endif
173
}
174
173
-template<>
175
+template <>
176
inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
177
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
178
assert(false);
@@ -179,12 +181,12 @@ inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
181
#endif
182
}
183
182
-template<>
184
+template <>
185
inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
186
return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
187
}
188
187
-template<>
189
+template <>
190
inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
191
bf16_4_t c;
192
c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -192,7 +194,7 @@ inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
194
return c;
195
}
196
195
-template<>
197
+template <>
198
inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
199
__nv_bfloat162 s = bf162bf162(a);
200
bf16_4_t c;
@@ -201,7 +203,7 @@ inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
203
return c;
204
}
205
204
-template<>
206
+template <>
207
inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
208
bf16_8_t c;
209
c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -211,7 +213,7 @@ inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
213
return c;
214
}
215
214
-template<>
216
+template <>
217
inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
218
__nv_bfloat162 s = bf162bf162(a);
219
bf16_8_t c;
@@ -222,26 +224,26 @@ inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
224
return c;
225
}
226
225
-template<>
227
+template <>
228
inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b) {
229
float fa = __bfloat162float(a);
230
float fb = __bfloat162float(b);
231
return fa * fb;
232
}
233
232
-template<>
234
+template <>
235
inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
236
float2 fa = bf1622float2(a);
237
float2 fb = bf1622float2(b);
238
return mul<float2, float2, float2>(fa, fb);
239
}
240
239
-template<>
241
+template <>
242
inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
243
return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
244
}
245
244
-template<>
246
+template <>
247
inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
248
Float4_ fc;
249
fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -249,7 +251,7 @@ inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
251
return fc;
252
}
253
252
-template<>
254
+template <>
255
inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
256
__nv_bfloat162 s = bf162bf162(a);
257
Float4_ fc;
@@ -258,7 +260,7 @@ inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
260
return fc;
261
}
262
261
-template<>
263
+template <>
264
inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
265
Float8_ fc;
266
fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
@@ -268,7 +270,7 @@ inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
270
return fc;
271
}
272
271
-template<>
273
+template <>
274
inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
275
__nv_bfloat162 s = bf162bf162(a);
276
Float8_ fc;
@@ -280,7 +282,8 @@ inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
282
}
283
284
// Vector fused multiply-add.
283
-inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
285
+inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b,
286
+ __nv_bfloat162 c) {
287
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
288
assert(false);
289
#else
@@ -288,7 +291,8 @@ inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bf
291
#endif
292
}
293
291
-inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
294
+inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b,
295
+ __nv_bfloat162 c) {
296
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
297
assert(false);
298
#else
@@ -379,23 +383,23 @@ inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc) {
383
}
384
385
// Vector sum.
382
-template<>
386
+template <>
387
inline __device__ float sum(__nv_bfloat16 v) {
388
return __bfloat162float(v);
389
}
390
387
-template<>
391
+template <>
392
inline __device__ float sum(__nv_bfloat162 v) {
393
float2 vf = bf1622float2(v);
394
return vf.x + vf.y;
395
}
396
393
-template<>
397
+template <>
398
inline __device__ float sum(bf16_4_t v) {
399
return sum(v.x) + sum(v.y);
400
}
401
398
-template<>
402
+template <>
403
inline __device__ float sum(bf16_8_t v) {
404
return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
405
}
@@ -448,4 +452,4 @@ inline __device__ void zero(__nv_bfloat16& dst) {
452
#endif
453
}
454
451
-} // namespace vllm
455
+} // namespace vllm
csrc/attention/dtype_float16.cuh
+47
-45
@@ -1,6 +1,8 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
- * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
+ * and
5
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6
* Copyright (c) 2023, The vLLM team.
7
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8
*
@@ -30,37 +32,37 @@
32
namespace vllm {
33
34
// FP16 vector types for Q, K, V.
33
-template<>
35
+template <>
36
struct Vec<uint16_t, 1> {
37
using Type = uint16_t;
38
};
37
-template<>
39
+template <>
40
struct Vec<uint16_t, 2> {
41
using Type = uint32_t;
42
};
41
-template<>
43
+template <>
44
struct Vec<uint16_t, 4> {
45
using Type = uint2;
46
};
45
-template<>
47
+template <>
48
struct Vec<uint16_t, 8> {
49
using Type = uint4;
50
};
51
52
// FP32 accumulator vector types corresponding to Vec.
51
-template<>
53
+template <>
54
struct FloatVec<uint16_t> {
55
using Type = float;
56
};
55
-template<>
57
+template <>
58
struct FloatVec<uint32_t> {
59
using Type = float2;
60
};
59
-template<>
61
+template <>
62
struct FloatVec<uint2> {
63
using Type = Float4_;
64
};
63
-template<>
65
+template <>
66
struct FloatVec<uint4> {
67
using Type = Float8_;
68
};
@@ -73,8 +75,8 @@ inline __device__ uint32_t h0_h0(uint16_t a) {
75
return b;
76
#else
77
union {
76
- uint32_t u32;
77
- uint16_t u16[2];
78
+ uint32_t u32;
79
+ uint16_t u16[2];
80
} tmp;
81
tmp.u16[0] = a;
82
tmp.u16[1] = a;
@@ -130,10 +132,12 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
132
} tmp;
133
#ifndef USE_ROCM
134
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
133
- asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
135
+ asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n"
136
+ : "=r"(tmp.u32)
137
+ : "f"(f.y), "f"(f.x));
138
#else
135
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
136
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
139
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
140
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
141
#endif
142
#else
143
tmp.u16[0] = float_to_half(f.x);
@@ -201,7 +205,7 @@ inline __device__ Float8_ add(uint4 a, Float8_ fb) {
205
}
206
207
// Vector multiplication.
204
-template<>
208
+template <>
209
inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
210
uint16_t c;
211
#ifndef USE_ROCM
@@ -212,7 +216,7 @@ inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
216
return c;
217
}
218
215
-template<>
219
+template <>
220
inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
221
uint32_t c;
222
#ifndef USE_ROCM
@@ -223,12 +227,12 @@ inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
227
return c;
228
}
229
226
-template<>
230
+template <>
231
inline __device__ uint32_t mul(uint16_t a, uint32_t b) {
232
return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
233
}
234
231
-template<>
235
+template <>
236
inline __device__ uint2 mul(uint2 a, uint2 b) {
237
uint2 c;
238
c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
@@ -236,7 +240,7 @@ inline __device__ uint2 mul(uint2 a, uint2 b) {
240
return c;
241
}
242
239
-template<>
243
+template <>
244
inline __device__ uint2 mul(uint16_t a, uint2 b) {
245
uint32_t s = h0_h0(a);
246
uint2 c;
@@ -245,7 +249,7 @@ inline __device__ uint2 mul(uint16_t a, uint2 b) {
249
return c;
250
}
251
248
-template<>
252
+template <>
253
inline __device__ uint4 mul(uint4 a, uint4 b) {
254
uint4 c;
255
c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
@@ -255,7 +259,7 @@ inline __device__ uint4 mul(uint4 a, uint4 b) {
259
return c;
260
}
261
258
-template<>
262
+template <>
263
inline __device__ uint4 mul(uint16_t a, uint4 b) {
264
uint32_t s = h0_h0(a);
265
uint4 c;
@@ -266,26 +270,26 @@ inline __device__ uint4 mul(uint16_t a, uint4 b) {
270
return c;
271
}
272
269
-template<>
273
+template <>
274
inline __device__ float mul(uint16_t a, uint16_t b) {
275
float fa = half_to_float(a);
276
float fb = half_to_float(b);
277
return fa * fb;
278
}
279
276
-template<>
280
+template <>
281
inline __device__ float2 mul(uint32_t a, uint32_t b) {
282
float2 fa = half2_to_float2(a);
283
float2 fb = half2_to_float2(b);
284
return mul<float2, float2, float2>(fa, fb);
285
}
286
283
-template<>
287
+template <>
288
inline __device__ float2 mul(uint16_t a, uint32_t b) {
289
return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
290
}
291
288
-template<>
292
+template <>
293
inline __device__ Float4_ mul(uint2 a, uint2 b) {
294
Float4_ fc;
295
fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
@@ -293,7 +297,7 @@ inline __device__ Float4_ mul(uint2 a, uint2 b) {
297
return fc;
298
}
299
296
-template<>
300
+template <>
301
inline __device__ Float4_ mul(uint16_t a, uint2 b) {
302
uint32_t s = h0_h0(a);
303
Float4_ fc;
@@ -302,7 +306,7 @@ inline __device__ Float4_ mul(uint16_t a, uint2 b) {
306
return fc;
307
}
308
305
-template<>
309
+template <>
310
inline __device__ Float8_ mul(uint4 a, uint4 b) {
311
Float8_ fc;
312
fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
@@ -312,7 +316,7 @@ inline __device__ Float8_ mul(uint4 a, uint4 b) {
316
return fc;
317
}
318
315
-template<>
319
+template <>
320
inline __device__ Float8_ mul(uint16_t a, uint4 b) {
321
uint32_t s = h0_h0(a);
322
Float8_ fc;
@@ -327,9 +331,13 @@ inline __device__ Float8_ mul(uint16_t a, uint4 b) {
331
inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
332
uint32_t d;
333
#ifndef USE_ROCM
330
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
334
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
335
+ : "=r"(d)
336
+ : "r"(a), "r"(b), "r"(c));
337
#else
332
- asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n" : "=v"(d) : "v"(a), "v"(b), "v"(c));
338
+ asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n"
339
+ : "=v"(d)
340
+ : "v"(a), "v"(b), "v"(c));
341
#endif
342
return d;
343
}
@@ -423,24 +431,24 @@ inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc) {
431
}
432
433
// Vector sum.
426
-template<>
434
+template <>
435
inline __device__ float sum(uint16_t v) {
436
return half_to_float(v);
437
}
438
431
-template<>
439
+template <>
440
inline __device__ float sum(uint32_t v) {
441
float2 tmp = half2_to_float2(v);
442
return tmp.x + tmp.y;
443
}
444
437
-template<>
445
+template <>
446
inline __device__ float sum(uint2 v) {
447
uint32_t c = add(v.x, v.y);
448
return sum(c);
449
}
450
443
-template<>
451
+template <>
452
inline __device__ float sum(uint4 v) {
453
uint32_t c = add(v.x, v.y);
454
c = add(c, v.z);
@@ -470,13 +478,9 @@ inline __device__ void from_float(uint4& dst, Float8_ src) {
478
}
479
480
// From float16 to float32.
473
-inline __device__ float to_float(uint16_t u) {
474
- return half_to_float(u);
475
-}
481
+inline __device__ float to_float(uint16_t u) { return half_to_float(u); }
482
477
-inline __device__ float2 to_float(uint32_t u) {
478
- return half2_to_float2(u);
479
-}
483
+inline __device__ float2 to_float(uint32_t u) { return half2_to_float2(u); }
484
485
inline __device__ Float4_ to_float(uint2 u) {
486
Float4_ tmp;
@@ -495,8 +499,6 @@ inline __device__ Float8_ to_float(uint4 u) {
499
}
500
501
// Zero-out a variable.
498
-inline __device__ void zero(uint16_t& dst) {
499
- dst = uint16_t(0);
500
-}
502
+inline __device__ void zero(uint16_t& dst) { dst = uint16_t(0); }
503
502
-} // namespace vllm
504
+} // namespace vllm
csrc/attention/dtype_float32.cuh
+33
-55
@@ -1,6 +1,8 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
3
- * and https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
+ * and
5
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h
6
* Copyright (c) 2023, The vLLM team.
7
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
8
*
@@ -38,37 +40,35 @@ struct Float8_ {
40
};
41
42
// FP32 vector types for Q, K, V.
41
-template<>
43
+template <>
44
struct Vec<float, 1> {
45
using Type = float;
46
};
45
-template<>
47
+template <>
48
struct Vec<float, 2> {
49
using Type = float2;
50
};
49
-template<>
51
+template <>
52
struct Vec<float, 4> {
53
using Type = float4;
54
};
55
56
// FP32 accumulator vector types corresponding to Vec.
55
-template<>
57
+template <>
58
struct FloatVec<float> {
59
using Type = float;
60
};
59
-template<>
61
+template <>
62
struct FloatVec<float2> {
63
using Type = float2;
64
};
63
-template<>
65
+template <>
66
struct FloatVec<float4> {
67
using Type = float4;
68
};
69
70
// Vector addition.
69
-inline __device__ float add(float a, float b) {
70
- return a + b;
71
-}
71
+inline __device__ float add(float a, float b) { return a + b; }
72
73
inline __device__ float2 add(float2 a, float2 b) {
74
float2 c;
@@ -87,12 +87,12 @@ inline __device__ float4 add(float4 a, float4 b) {
87
}
88
89
// Vector multiplication.
90
-template<>
90
+template <>
91
inline __device__ float mul<float, float>(float a, float b) {
92
return a * b;
93
}
94
95
-template<>
95
+template <>
96
inline __device__ float2 mul(float2 a, float2 b) {
97
float2 c;
98
c.x = a.x * b.x;
@@ -100,7 +100,7 @@ inline __device__ float2 mul(float2 a, float2 b) {
100
return c;
101
}
102
103
-template<>
103
+template <>
104
inline __device__ float2 mul(float a, float2 b) {
105
float2 c;
106
c.x = a * b.x;
@@ -108,7 +108,7 @@ inline __device__ float2 mul(float a, float2 b) {
108
return c;
109
}
110
111
-template<>
111
+template <>
112
inline __device__ float4 mul(float4 a, float4 b) {
113
float4 c;
114
c.x = a.x * b.x;
@@ -118,7 +118,7 @@ inline __device__ float4 mul(float4 a, float4 b) {
118
return c;
119
}
120
121
-template<>
121
+template <>
122
inline __device__ float4 mul(float a, float4 b) {
123
float4 c;
124
c.x = a * b.x;
@@ -129,9 +129,7 @@ inline __device__ float4 mul(float a, float4 b) {
129
}
130
131
// Vector fused multiply-add.
132
-inline __device__ float fma(float a, float b, float c) {
133
- return a * b + c;
134
-}
132
+inline __device__ float fma(float a, float b, float c) { return a * b + c; }
133
134
inline __device__ float2 fma(float2 a, float2 b, float2 c) {
135
float2 d;
@@ -182,35 +180,33 @@ inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c) {
180
}
181
182
// Vector sum.
185
-template<>
183
+template <>
184
inline __device__ float sum(float v) {
185
return v;
186
}
187
190
-template<>
188
+template <>
189
inline __device__ float sum(float2 v) {
190
return v.x + v.y;
191
}
192
195
-template<>
193
+template <>
194
inline __device__ float sum(float4 v) {
195
return v.x + v.y + v.z + v.w;
196
}
197
200
-template<>
198
+template <>
199
inline __device__ float sum(Float4_ v) {
200
return v.x.x + v.x.y + v.y.x + v.y.y;
201
}
202
205
-template<>
203
+template <>
204
inline __device__ float sum(Float8_ v) {
205
return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
206
}
207
208
// Vector dot product.
211
-inline __device__ float dot(float a, float b) {
212
- return a * b;
213
-}
209
+inline __device__ float dot(float a, float b) { return a * b; }
210
211
inline __device__ float dot(float2 a, float2 b) {
212
float2 c = mul<float2, float2, float2>(a, b);
@@ -232,42 +228,24 @@ inline __device__ float dot(Float8_ a, Float8_ b) {
228
}
229
230
// From float to float.
235
-inline __device__ void from_float(float& dst, float src) {
236
- dst = src;
237
-}
231
+inline __device__ void from_float(float& dst, float src) { dst = src; }
232
239
-inline __device__ void from_float(float2& dst, float2 src) {
240
- dst = src;
241
-}
233
+inline __device__ void from_float(float2& dst, float2 src) { dst = src; }
234
243
-inline __device__ void from_float(float4& dst, float4 src) {
244
- dst = src;
245
-}
235
+inline __device__ void from_float(float4& dst, float4 src) { dst = src; }
236
237
// From float to float.
248
-inline __device__ float to_float(float u) {
249
- return u;
250
-}
238
+inline __device__ float to_float(float u) { return u; }
239
252
-inline __device__ float2 to_float(float2 u) {
253
- return u;
254
-}
240
+inline __device__ float2 to_float(float2 u) { return u; }
241
256
-inline __device__ float4 to_float(float4 u) {
257
- return u;
258
-}
242
+inline __device__ float4 to_float(float4 u) { return u; }
243
260
-inline __device__ Float4_ to_float(Float4_ u) {
261
- return u;
262
-}
244
+inline __device__ Float4_ to_float(Float4_ u) { return u; }
245
264
-inline __device__ Float8_ to_float(Float8_ u) {
265
- return u;
266
-}
246
+inline __device__ Float8_ to_float(Float8_ u) { return u; }
247
248
// Zero-out a variable.
269
-inline __device__ void zero(float& dst) {
270
- dst = 0.f;
271
-}
249
+inline __device__ void zero(float& dst) { dst = 0.f; }
250
273
-} // namespace vllm
251
+} // namespace vllm
csrc/attention/dtype_fp8.cuh
+16
-16
@@ -4,38 +4,38 @@
4
5
#include <stdint.h>
6
#ifdef ENABLE_FP8
7
-#ifndef USE_ROCM
8
-#include <cuda_fp8.h>
9
-#endif // USE_ROCM
10
-#endif // ENABLE_FP8
7
+ #ifndef USE_ROCM
8
+ #include <cuda_fp8.h>
9
+ #endif // USE_ROCM
10
+#endif // ENABLE_FP8
11
12
namespace vllm {
13
14
enum class Fp8KVCacheDataType {
15
- kAuto = 0,
16
- kFp8E4M3 = 1,
17
- kFp8E5M2 = 2,
15
+ kAuto = 0,
16
+ kFp8E4M3 = 1,
17
+ kFp8E5M2 = 2,
18
};
19
20
// fp8 vector types for quantization of kv cache
21
-template<>
21
+template <>
22
struct Vec<uint8_t, 1> {
23
- using Type = uint8_t;
23
+ using Type = uint8_t;
24
};
25
26
-template<>
26
+template <>
27
struct Vec<uint8_t, 2> {
28
- using Type = uint16_t;
28
+ using Type = uint16_t;
29
};
30
31
-template<>
31
+template <>
32
struct Vec<uint8_t, 4> {
33
- using Type = uint32_t;
33
+ using Type = uint32_t;
34
};
35
36
-template<>
36
+template <>
37
struct Vec<uint8_t, 8> {
38
- using Type = uint2;
38
+ using Type = uint2;
39
};
40
41
-} // namespace vllm
41
+} // namespace vllm
csrc/cache.h
+16
-28
@@ -5,36 +5,24 @@
5
#include <map>
6
#include <vector>
7
8
-void swap_blocks(
9
- torch::Tensor& src,
10
- torch::Tensor& dst,
11
- const torch::Tensor& block_mapping);
8
+void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
9
+ const torch::Tensor& block_mapping);
10
13
-void copy_blocks(
14
- std::vector<torch::Tensor>& key_caches,
15
- std::vector<torch::Tensor>& value_caches,
16
- const torch::Tensor& block_mapping);
11
+void copy_blocks(std::vector<torch::Tensor>& key_caches,
12
+ std::vector<torch::Tensor>& value_caches,
13
+ const torch::Tensor& block_mapping);
14
18
-void reshape_and_cache(
19
- torch::Tensor& key,
20
- torch::Tensor& value,
21
- torch::Tensor& key_cache,
22
- torch::Tensor& value_cache,
23
- torch::Tensor& slot_mapping,
24
- const std::string& kv_cache_dtype,
25
- const float kv_scale);
15
+void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
16
+ torch::Tensor& key_cache, torch::Tensor& value_cache,
17
+ torch::Tensor& slot_mapping,
18
+ const std::string& kv_cache_dtype, const float kv_scale);
19
27
-void reshape_and_cache_flash(
28
- torch::Tensor& key,
29
- torch::Tensor& value,
30
- torch::Tensor& key_cache,
31
- torch::Tensor& value_cache,
32
- torch::Tensor& slot_mapping,
33
- const std::string& kv_cache_dtype);
20
+void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
21
+ torch::Tensor& key_cache,
22
+ torch::Tensor& value_cache,
23
+ torch::Tensor& slot_mapping,
24
+ const std::string& kv_cache_dtype);
25
26
// Just for unittest
36
-void convert_fp8(
37
- torch::Tensor& dst_cache,
38
- torch::Tensor& src_cache,
39
- const float scale,
40
- const std::string& kv_cache_dtype);
27
+void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
28
+ const float scale, const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+131
-157
@@ -6,9 +6,9 @@
6
#include "dispatch_utils.h"
7
8
#ifdef USE_ROCM
9
-#include "quantization/fp8/amd/quant_utils.cuh"
9
+ #include "quantization/fp8/amd/quant_utils.cuh"
10
#else
11
-#include "quantization/fp8/nvidia/quant_utils.cuh"
11
+ #include "quantization/fp8/nvidia/quant_utils.cuh"
12
#endif
13
14
#include <algorithm>
@@ -18,20 +18,17 @@
18
19
#ifdef USE_ROCM
20
#include <hip/hip_bf16.h>
21
- typedef __hip_bfloat16 __nv_bfloat16;
21
+typedef __hip_bfloat16 __nv_bfloat16;
22
#endif
23
24
-void swap_blocks(
25
- torch::Tensor& src,
26
- torch::Tensor& dst,
27
- const torch::Tensor& block_mapping) {
24
+void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
25
+ const torch::Tensor& block_mapping) {
26
torch::Device src_device = src.device();
27
torch::Device dst_device = dst.device();
28
cudaMemcpyKind memcpy_type;
29
if (src_device.is_cuda() && dst_device.is_cuda()) {
32
- TORCH_CHECK(
33
- src_device.index() == dst_device.index(),
34
- "src and dst must be on the same GPU");
30
+ TORCH_CHECK(src_device.index() == dst_device.index(),
31
+ "src and dst must be on the same GPU");
32
memcpy_type = cudaMemcpyDeviceToDevice;
33
} else if (src_device.is_cuda() && dst_device.is_cpu()) {
34
memcpy_type = cudaMemcpyDeviceToHost;
@@ -41,16 +38,17 @@ void swap_blocks(
38
TORCH_CHECK(false, "Invalid device combination");
39
}
40
44
- // NOTE(youkaichao): keep in mind that `block_mapping` should be
41
+ // NOTE(youkaichao): keep in mind that `block_mapping` should be
42
// a cpu tensor, otherwise every `item` call will require a gpu-cpu
43
// synchronization.
44
TORCH_CHECK(block_mapping.device().is_cpu(), "block_mapping must be on CPU");
45
49
- char *src_ptr = static_cast<char*>(src.data_ptr());
50
- char *dst_ptr = static_cast<char*>(dst.data_ptr());
46
+ char* src_ptr = static_cast<char*>(src.data_ptr());
47
+ char* dst_ptr = static_cast<char*>(dst.data_ptr());
48
49
const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
53
- const at::cuda::OptionalCUDAGuard device_guard(src_device.is_cuda() ? src_device : dst_device);
50
+ const at::cuda::OptionalCUDAGuard device_guard(
51
+ src_device.is_cuda() ? src_device : dst_device);
52
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
53
// NOTE(woosuk): This can be slow if the number of blocks is large.
54
const int64_t num_blocks = block_mapping.size(0);
@@ -59,29 +57,25 @@ void swap_blocks(
57
int64_t dst_block_number = block_mapping[i][1].item<int64_t>();
58
int64_t src_offset = src_block_number * block_size_in_bytes;
59
int64_t dst_offset = dst_block_number * block_size_in_bytes;
62
- cudaMemcpyAsync(
63
- dst_ptr + dst_offset,
64
- src_ptr + src_offset,
65
- block_size_in_bytes,
66
- memcpy_type,
67
- stream);
60
+ cudaMemcpyAsync(dst_ptr + dst_offset, src_ptr + src_offset,
61
+ block_size_in_bytes, memcpy_type, stream);
62
}
63
}
64
65
namespace vllm {
66
67
// Grid: (num_layers, num_pairs)
74
-template<typename scalar_t>
75
-__global__ void copy_blocks_kernel(
76
- int64_t* key_cache_ptrs,
77
- int64_t* value_cache_ptrs,
78
- const int64_t* __restrict__ block_mapping,
79
- const int numel_per_block) {
68
+template <typename scalar_t>
69
+__global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
70
+ int64_t* value_cache_ptrs,
71
+ const int64_t* __restrict__ block_mapping,
72
+ const int numel_per_block) {
73
const int layer_idx = blockIdx.x;
74
const int pair_idx = blockIdx.y;
75
76
scalar_t* key_cache = reinterpret_cast<scalar_t*>(key_cache_ptrs[layer_idx]);
84
- scalar_t* value_cache = reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
77
+ scalar_t* value_cache =
78
+ reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
79
int64_t src_block_number = block_mapping[2 * pair_idx];
80
int64_t dst_block_number = block_mapping[2 * pair_idx + 1];
81
@@ -99,12 +93,11 @@ __global__ void copy_blocks_kernel(
93
}
94
}
95
102
-} // namespace vllm
96
+} // namespace vllm
97
104
-void copy_blocks(
105
- std::vector<torch::Tensor>& key_caches,
106
- std::vector<torch::Tensor>& value_caches,
107
- const torch::Tensor& block_mapping) {
98
+void copy_blocks(std::vector<torch::Tensor>& key_caches,
99
+ std::vector<torch::Tensor>& value_caches,
100
+ const torch::Tensor& block_mapping) {
101
int num_layers = key_caches.size();
102
TORCH_CHECK(num_layers == value_caches.size());
103
if (num_layers == 0) {
@@ -118,8 +111,10 @@ void copy_blocks(
111
int64_t key_cache_ptrs[num_layers];
112
int64_t value_cache_ptrs[num_layers];
113
for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
121
- key_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
122
- value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
114
+ key_cache_ptrs[layer_idx] =
115
+ reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
116
+ value_cache_ptrs[layer_idx] =
117
+ reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
118
}
119
120
// block_mapping is a 2D tensor with shape (num_pairs, 2).
@@ -127,10 +122,12 @@ void copy_blocks(
122
123
// Move the data structures to the GPU.
124
// NOTE: This synchronizes the CPU and GPU.
130
- torch::Tensor key_cache_ptrs_tensor = torch::from_blob(
131
- key_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
132
- torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
133
- value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
125
+ torch::Tensor key_cache_ptrs_tensor =
126
+ torch::from_blob(key_cache_ptrs, {num_layers}, torch::kInt64)
127
+ .to(cache_device);
128
+ torch::Tensor value_cache_ptrs_tensor =
129
+ torch::from_blob(value_cache_ptrs, {num_layers}, torch::kInt64)
130
+ .to(cache_device);
131
132
// Launch the kernel.
133
const int numel_per_block = key_caches[0][0].numel();
@@ -139,31 +136,28 @@ void copy_blocks(
136
const at::cuda::OptionalCUDAGuard device_guard(cache_device);
137
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
138
VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
142
- key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
143
- vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
144
- key_cache_ptrs_tensor.data_ptr<int64_t>(),
145
- value_cache_ptrs_tensor.data_ptr<int64_t>(),
146
- block_mapping.data_ptr<int64_t>(),
147
- numel_per_block);
148
- }));
139
+ key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
140
+ vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
141
+ key_cache_ptrs_tensor.data_ptr<int64_t>(),
142
+ value_cache_ptrs_tensor.data_ptr<int64_t>(),
143
+ block_mapping.data_ptr<int64_t>(), numel_per_block);
144
+ }));
145
}
146
147
namespace vllm {
148
153
-template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
149
+template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
150
__global__ void reshape_and_cache_kernel(
155
- const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
156
- const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
157
- cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
158
- cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
159
- const int64_t* __restrict__ slot_mapping, // [num_tokens]
160
- const int key_stride,
161
- const int value_stride,
162
- const int num_heads,
163
- const int head_size,
164
- const int block_size,
165
- const int x,
166
- const float kv_scale) {
151
+ const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
152
+ const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
153
+ cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x,
154
+ // block_size, x]
155
+ cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size,
156
+ // block_size]
157
+ const int64_t* __restrict__ slot_mapping, // [num_tokens]
158
+ const int key_stride, const int value_stride, const int num_heads,
159
+ const int head_size, const int block_size, const int x,
160
+ const float kv_scale) {
161
const int64_t token_idx = blockIdx.x;
162
const int64_t slot_idx = slot_mapping[token_idx];
163
if (slot_idx < 0) {
@@ -184,40 +178,39 @@ __global__ void reshape_and_cache_kernel(
178
const int x_idx = head_offset / x;
179
const int x_offset = head_offset % x;
180
187
- const int64_t tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
188
- + head_idx * (head_size / x) * block_size * x
189
- + x_idx * block_size * x
190
- + block_offset * x
191
- + x_offset;
192
- const int64_t tgt_value_idx = block_idx * num_heads * head_size * block_size
193
- + head_idx * head_size * block_size
194
- + head_offset * block_size
195
- + block_offset;
181
+ const int64_t tgt_key_idx =
182
+ block_idx * num_heads * (head_size / x) * block_size * x +
183
+ head_idx * (head_size / x) * block_size * x + x_idx * block_size * x +
184
+ block_offset * x + x_offset;
185
+ const int64_t tgt_value_idx =
186
+ block_idx * num_heads * head_size * block_size +
187
+ head_idx * head_size * block_size + head_offset * block_size +
188
+ block_offset;
189
scalar_t tgt_key = key[src_key_idx];
190
scalar_t tgt_value = value[src_value_idx];
191
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
192
key_cache[tgt_key_idx] = tgt_key;
193
value_cache[tgt_value_idx] = tgt_value;
194
} else {
202
- key_cache[tgt_key_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
203
- value_cache[tgt_value_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
195
+ key_cache[tgt_key_idx] =
196
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
197
+ value_cache[tgt_value_idx] =
198
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
199
}
200
}
201
}
202
208
-template<typename scalar_t>
203
+template <typename scalar_t>
204
__global__ void reshape_and_cache_flash_kernel(
210
- const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
211
- const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
212
- scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads, head_size]
213
- scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads, head_size]
214
- const int64_t* __restrict__ slot_mapping, // [num_tokens]
215
- const int block_stride,
216
- const int key_stride,
217
- const int value_stride,
218
- const int num_heads,
219
- const int head_size,
220
- const int block_size) {
205
+ const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
206
+ const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
207
+ scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads,
208
+ // head_size]
209
+ scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads,
210
+ // head_size]
211
+ const int64_t* __restrict__ slot_mapping, // [num_tokens]
212
+ const int block_stride, const int key_stride, const int value_stride,
213
+ const int num_heads, const int head_size, const int block_size) {
214
const int64_t token_idx = blockIdx.x;
215
const int64_t slot_idx = slot_mapping[token_idx];
216
// NOTE: slot_idx can be -1 if the token is padded
@@ -232,43 +225,37 @@ __global__ void reshape_and_cache_flash_kernel(
225
const int64_t src_value_idx = token_idx * value_stride + i;
226
const int head_idx = i / head_size;
227
const int head_offset = i % head_size;
235
- const int64_t tgt_value_idx = block_idx * block_stride
236
- + block_offset * num_heads * head_size
237
- + head_idx * head_size
238
- + head_offset;
228
+ const int64_t tgt_value_idx = block_idx * block_stride +
229
+ block_offset * num_heads * head_size +
230
+ head_idx * head_size + head_offset;
231
k_cache[tgt_value_idx] = key[src_key_idx];
232
v_cache[tgt_value_idx] = value[src_value_idx];
233
}
234
}
243
-} // namespace vllm
235
+} // namespace vllm
236
237
// KV_T is the stored data type of kv-cache.
238
// CACHE_T is the data type of key and value tensors.
239
// KV_DTYPE is the real data type of kv-cache.
248
-#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
249
- vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE><<<grid, block, 0, stream>>>( \
250
- reinterpret_cast<KV_T*>(key.data_ptr()), \
251
- reinterpret_cast<KV_T*>(value.data_ptr()), \
252
- reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
253
- reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
254
- slot_mapping.data_ptr<int64_t>(), \
255
- key_stride, \
256
- value_stride, \
257
- num_heads, \
258
- head_size, \
259
- block_size, \
260
- x, \
261
- kv_scale);
240
+#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
241
+ vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE> \
242
+ <<<grid, block, 0, stream>>>( \
243
+ reinterpret_cast<KV_T*>(key.data_ptr()), \
244
+ reinterpret_cast<KV_T*>(value.data_ptr()), \
245
+ reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
246
+ reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
247
+ slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
248
+ num_heads, head_size, block_size, x, kv_scale);
249
250
void reshape_and_cache(
264
- torch::Tensor& key, // [num_tokens, num_heads, head_size]
265
- torch::Tensor& value, // [num_tokens, num_heads, head_size]
266
- torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
267
- torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
268
- torch::Tensor& slot_mapping, // [num_tokens]
269
- const std::string& kv_cache_dtype,
270
- const float kv_scale)
271
-{
251
+ torch::Tensor& key, // [num_tokens, num_heads, head_size]
252
+ torch::Tensor& value, // [num_tokens, num_heads, head_size]
253
+ torch::Tensor&
254
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
255
+ torch::Tensor&
256
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
257
+ torch::Tensor& slot_mapping, // [num_tokens]
258
+ const std::string& kv_cache_dtype, const float kv_scale) {
259
int num_tokens = key.size(0);
260
int num_heads = key.size(1);
261
int head_size = key.size(2);
@@ -283,17 +270,17 @@ void reshape_and_cache(
270
const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
271
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
272
286
- DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype, CALL_RESHAPE_AND_CACHE)
273
+ DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype,
274
+ CALL_RESHAPE_AND_CACHE)
275
}
276
277
void reshape_and_cache_flash(
290
- torch::Tensor& key, // [num_tokens, num_heads, head_size]
291
- torch::Tensor& value, // [num_tokens, num_heads, head_size]
292
- torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
293
- torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
294
- torch::Tensor& slot_mapping, // [num_tokens]
295
- const std::string& kv_cache_dtype)
296
-{
278
+ torch::Tensor& key, // [num_tokens, num_heads, head_size]
279
+ torch::Tensor& value, // [num_tokens, num_heads, head_size]
280
+ torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
281
+ torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
282
+ torch::Tensor& slot_mapping, // [num_tokens]
283
+ const std::string& kv_cache_dtype) {
284
// FIXME: only support auto datatype, does not support fp8
285
if (kv_cache_dtype != "auto") {
286
TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
@@ -313,62 +300,47 @@ void reshape_and_cache_flash(
300
const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
301
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
302
VLLM_DISPATCH_FLOATING_TYPES(
316
- key.scalar_type(),
317
- "reshape_and_cache_flash",
318
- [&] {
319
- vllm::reshape_and_cache_flash_kernel<scalar_t><<<grid, block, 0, stream>>>(
320
- key.data_ptr<scalar_t>(),
321
- value.data_ptr<scalar_t>(),
322
- k_cache.data_ptr<scalar_t>(),
323
- v_cache.data_ptr<scalar_t>(),
324
- slot_mapping.data_ptr<int64_t>(),
325
- block_stride,
326
- key_stride,
327
- value_stride,
328
- num_heads,
329
- head_size,
330
- block_size);
331
- });
303
+ key.scalar_type(), "reshape_and_cache_flash", [&] {
304
+ vllm::reshape_and_cache_flash_kernel<scalar_t>
305
+ <<<grid, block, 0, stream>>>(
306
+ key.data_ptr<scalar_t>(), value.data_ptr<scalar_t>(),
307
+ k_cache.data_ptr<scalar_t>(), v_cache.data_ptr<scalar_t>(),
308
+ slot_mapping.data_ptr<int64_t>(), block_stride, key_stride,
309
+ value_stride, num_heads, head_size, block_size);
310
+ });
311
}
312
313
namespace vllm {
314
336
-template<typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
337
-__global__ void convert_fp8_kernel(
338
- const Tin* __restrict__ src_cache,
339
- Tout* __restrict__ dst_cache,
340
- const float kv_scale,
341
- const int64_t block_stride) {
315
+template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
316
+__global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
317
+ Tout* __restrict__ dst_cache,
318
+ const float kv_scale,
319
+ const int64_t block_stride) {
320
const int64_t block_idx = blockIdx.x;
321
for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
322
int64_t idx = block_idx * block_stride + i;
345
- dst_cache[idx] = fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
323
+ dst_cache[idx] =
324
+ fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
325
}
326
}
327
349
-} // namespace vllm
328
+} // namespace vllm
329
351
-#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
352
- vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
353
- reinterpret_cast<Tin*>(src_cache.data_ptr()), \
354
- reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
355
- kv_scale, \
356
- block_stride);
330
+#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
331
+ vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
332
+ reinterpret_cast<Tin*>(src_cache.data_ptr()), \
333
+ reinterpret_cast<Tout*>(dst_cache.data_ptr()), kv_scale, block_stride);
334
335
// Only for testing.
359
-void convert_fp8(
360
- torch::Tensor& dst_cache,
361
- torch::Tensor& src_cache,
362
- const float kv_scale,
363
- const std::string& kv_cache_dtype)
364
-{
336
+void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
337
+ const float kv_scale, const std::string& kv_cache_dtype) {
338
torch::Device src_device = src_cache.device();
339
torch::Device dst_device = dst_cache.device();
340
TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
341
TORCH_CHECK(dst_device.is_cuda(), "dst must be on a GPU")
369
- TORCH_CHECK(
370
- src_device.index() == dst_device.index(),
371
- "src and dst must be on the same GPU");
342
+ TORCH_CHECK(src_device.index() == dst_device.index(),
343
+ "src and dst must be on the same GPU");
344
at::cuda::OptionalCUDAGuard device_guard(src_device);
345
346
int64_t num_blocks = src_cache.size(0);
@@ -398,13 +370,15 @@ void convert_fp8(
370
} else if (src_cache.dtype() == at::ScalarType::Half) {
371
CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
372
} else if (src_cache.dtype() == at::ScalarType::BFloat16) {
401
- CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kFp8E4M3);
373
+ CALL_CONVERT_FP8(uint8_t, __nv_bfloat16,
374
+ vllm::Fp8KVCacheDataType::kFp8E4M3);
375
} else if (dst_cache.dtype() == at::ScalarType::Float) {
376
CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
377
} else if (dst_cache.dtype() == at::ScalarType::Half) {
378
CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
379
} else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
407
- CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
380
+ CALL_CONVERT_FP8(__nv_bfloat16, uint8_t,
381
+ vllm::Fp8KVCacheDataType::kFp8E4M3);
382
}
383
} else {
384
TORCH_CHECK(false, "Unsupported data type: ", kv_cache_dtype);
csrc/cpu/activation.cpp
+28
-32
@@ -1,10 +1,10 @@
1
#include "cpu_types.hpp"
2
3
namespace {
4
-template <typename scalar_t, vec_op::FP32Vec8 (*func)(const vec_op::FP32Vec8 &),
4
+template <typename scalar_t, vec_op::FP32Vec8 (*func)(const vec_op::FP32Vec8&),
5
bool is_gated>
6
-void activation_kernel(int num_tokens, int d, scalar_t *__restrict__ input,
7
- scalar_t *__restrict__ output) {
6
+void activation_kernel(int num_tokens, int d, scalar_t* __restrict__ input,
7
+ scalar_t* __restrict__ output) {
8
using scalar_vec_t = vec_op::vec_t<scalar_t>;
9
constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
10
@@ -34,13 +34,13 @@ void activation_kernel(int num_tokens, int d, scalar_t *__restrict__ input,
34
}
35
}
36
37
-FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8 &x) {
37
+FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8& x) {
38
const vec_op::FP32Vec8 zeros(0.0);
39
const vec_op::FP32Vec8 ones(1.0);
40
return x / (ones + (zeros - x).exp());
41
}
42
43
-FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8 &x) {
43
+FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8& x) {
44
const vec_op::FP32Vec8 ones(1.0);
45
const vec_op::FP32Vec8 w1(0.79788456f);
46
const vec_op::FP32Vec8 w2(0.044715f);
@@ -50,7 +50,7 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8 &x) {
50
return w3 * x * (ones + t);
51
}
52
53
-FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8 &x) {
53
+FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) {
54
const vec_op::FP32Vec8 ones(1.0);
55
const vec_op::FP32Vec8 w1(0.79788456f);
56
const vec_op::FP32Vec8 w2(0.044715f);
@@ -59,14 +59,14 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8 &x) {
59
return w3 * x * (ones + t);
60
}
61
62
-FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8 &x) {
62
+FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) {
63
const vec_op::FP32Vec8 ones(1.0);
64
const vec_op::FP32Vec8 w1(M_SQRT1_2);
65
const vec_op::FP32Vec8 w2(0.5);
66
return x * w2 * (ones + (x * w1).er());
67
}
68
69
-FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8 &x) {
69
+FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8& x) {
70
const vec_op::FP32Vec8 ones(1.0);
71
const vec_op::FP32Vec8 w1(M_SQRT2 * M_2_SQRTPI * 0.5);
72
const vec_op::FP32Vec8 w2(0.5);
@@ -75,40 +75,36 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8 &x) {
75
const vec_op::FP32Vec8 inner = w1 * (x + x_3 * w3);
76
return x * w2 * (ones + inner.tanh());
77
}
78
-}; // namespace
78
+}; // namespace
79
80
-void silu_and_mul(torch::Tensor &out, torch::Tensor &input) {
80
+void silu_and_mul(torch::Tensor& out, torch::Tensor& input) {
81
int num_tokens = input.numel() / input.size(-1);
82
int d = input.size(-1) / 2;
83
84
- VLLM_DISPATCH_FLOATING_TYPES(
85
- input.scalar_type(), "silu_and_mul_impl", [&] {
86
- CPU_KERNEL_GUARD_IN(silu_and_mul_impl)
87
- activation_kernel<scalar_t, silu_act, true>(num_tokens, d,
88
- input.data_ptr<scalar_t>(),
89
- out.data_ptr<scalar_t>());
90
- CPU_KERNEL_GUARD_OUT(silu_and_mul_impl)
91
- });
84
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "silu_and_mul_impl", [&] {
85
+ CPU_KERNEL_GUARD_IN(silu_and_mul_impl)
86
+ activation_kernel<scalar_t, silu_act, true>(
87
+ num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
88
+ CPU_KERNEL_GUARD_OUT(silu_and_mul_impl)
89
+ });
90
}
91
94
-void gelu_and_mul(torch::Tensor &out, // [..., d]
95
- torch::Tensor &input) // [..., 2 * d]
92
+void gelu_and_mul(torch::Tensor& out, // [..., d]
93
+ torch::Tensor& input) // [..., 2 * d]
94
{
95
int num_tokens = input.numel() / input.size(-1);
96
int d = input.size(-1) / 2;
97
100
- VLLM_DISPATCH_FLOATING_TYPES(
101
- input.scalar_type(), "gelu_and_mul_impl", [&] {
102
- CPU_KERNEL_GUARD_IN(gelu_and_mul_impl)
103
- activation_kernel<scalar_t, gelu_act, true>(num_tokens, d,
104
- input.data_ptr<scalar_t>(),
105
- out.data_ptr<scalar_t>());
106
- CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl)
107
- });
98
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul_impl", [&] {
99
+ CPU_KERNEL_GUARD_IN(gelu_and_mul_impl)
100
+ activation_kernel<scalar_t, gelu_act, true>(
101
+ num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
102
+ CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl)
103
+ });
104
}
105
110
-void gelu_tanh_and_mul(torch::Tensor &out, // [..., d]
111
- torch::Tensor &input) // [..., 2 * d]
106
+void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
107
+ torch::Tensor& input) // [..., 2 * d]
108
{
109
int num_tokens = input.numel() / input.size(-1);
110
int d = input.size(-1) / 2;
@@ -123,7 +119,7 @@ void gelu_tanh_and_mul(torch::Tensor &out, // [..., d]
119
});
120
}
121
126
-void gelu_new(torch::Tensor &out, torch::Tensor &input) {
122
+void gelu_new(torch::Tensor& out, torch::Tensor& input) {
123
int num_tokens = input.numel() / input.size(-1);
124
int d = input.size(-1);
125
@@ -135,7 +131,7 @@ void gelu_new(torch::Tensor &out, torch::Tensor &input) {
131
});
132
}
133
138
-void gelu_fast(torch::Tensor &out, torch::Tensor &input) {
134
+void gelu_fast(torch::Tensor& out, torch::Tensor& input) {
135
int num_tokens = input.numel() / input.size(-1);
136
int d = input.size(-1);
137
csrc/cpu/attention.cpp
+205
-206
@@ -2,7 +2,8 @@
2
3
namespace {
4
5
-template <typename scalar_t> struct KernelVecType {
5
+template <typename scalar_t>
6
+struct KernelVecType {
7
using q_load_vec_type = void;
8
using q_vec_type = void;
9
using k_load_vec_type = void;
@@ -11,7 +12,8 @@ template <typename scalar_t> struct KernelVecType {
12
using v_load_vec_type = void;
13
};
14
14
-template <> struct KernelVecType<float> {
15
+template <>
16
+struct KernelVecType<float> {
17
using q_load_vec_type = vec_op::FP32Vec4;
18
using q_vec_type = vec_op::FP32Vec16;
19
using k_load_vec_type = vec_op::FP32Vec16;
@@ -21,7 +23,8 @@ template <> struct KernelVecType<float> {
23
};
24
25
#ifdef __AVX512BF16__
24
-template <> struct KernelVecType<c10::BFloat16> {
26
+template <>
27
+struct KernelVecType<c10::BFloat16> {
28
using q_load_vec_type = vec_op::BF16Vec8;
29
using q_vec_type = vec_op::BF16Vec32;
30
using k_load_vec_type = vec_op::BF16Vec32;
@@ -30,7 +33,8 @@ template <> struct KernelVecType<c10::BFloat16> {
33
using v_load_vec_type = vec_op::BF16Vec16;
34
};
35
#else
33
-template <> struct KernelVecType<c10::BFloat16> {
36
+template <>
37
+struct KernelVecType<c10::BFloat16> {
38
using q_load_vec_type = vec_op::BF16Vec8;
39
using q_vec_type = vec_op::FP32Vec16;
40
using k_load_vec_type = vec_op::BF16Vec16;
@@ -41,7 +45,7 @@ template <> struct KernelVecType<c10::BFloat16> {
45
#endif
46
47
template <typename T>
44
-FORCE_INLINE std::pair<T, T> reduceSoftmax(T *data, const int size,
48
+FORCE_INLINE std::pair<T, T> reduceSoftmax(T* data, const int size,
49
const int capacity) {
50
T max = data[0];
51
for (int i = 1; i < size; ++i) {
@@ -67,10 +71,11 @@ FORCE_INLINE std::pair<T, T> reduceSoftmax(T *data, const int size,
71
}
72
73
template <typename T>
70
-FORCE_INLINE std::pair<T, T>
71
-reduceSoftmaxAlibi(T *data, const int size, const int capacity,
72
- const float alibi_slope, const int start_index,
73
- const int seq_len) {
74
+FORCE_INLINE std::pair<T, T> reduceSoftmaxAlibi(T* data, const int size,
75
+ const int capacity,
76
+ const float alibi_slope,
77
+ const int start_index,
78
+ const int seq_len) {
79
data[0] += alibi_slope * (start_index - seq_len + 1);
80
T max = data[0];
81
for (int i = 1; i < size; ++i) {
@@ -98,7 +103,7 @@ reduceSoftmaxAlibi(T *data, const int size, const int capacity,
103
}
104
105
template <typename T>
101
-FORCE_INLINE void reducePartitonSoftmax(const T *max_data, T *sum_data,
106
+FORCE_INLINE void reducePartitonSoftmax(const T* max_data, T* sum_data,
107
const int size) {
108
T max = max_data[0];
109
for (int i = 1; i < size; ++i) {
@@ -132,9 +137,9 @@ struct reduceQKBlockKernel {
137
static_assert(k_load_vec_type::get_elem_num() % x == 0);
138
static_assert(q_load_vec_type::get_elem_num() * sizeof(scalar_t) == 16);
139
135
- FORCE_INLINE static void call(const scalar_t *__restrict__ q,
136
- const scalar_t *__restrict__ k_block,
137
- float *__restrict__ logits, float scale,
140
+ FORCE_INLINE static void call(const scalar_t* __restrict__ q,
141
+ const scalar_t* __restrict__ k_block,
142
+ float* __restrict__ logits, float scale,
143
const int token_num) {
144
const int group_num = (token_num + TOKEN_PER_GROUP - 1) / TOKEN_PER_GROUP;
145
@@ -196,8 +201,8 @@ struct reduceQKBlockKernel {
201
202
template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE,
203
int HEAD_PARTITION_SIZE, typename acc_t>
199
-FORCE_INLINE void reduceValueBlock(const float *prob, const scalar_t *v_block,
200
- acc_t &&acc) {
204
+FORCE_INLINE void reduceValueBlock(const float* prob, const scalar_t* v_block,
205
+ acc_t&& acc) {
206
using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
207
constexpr int ELEM_NUM = v_load_vec_type::get_elem_num();
208
static_assert(BLOCK_SIZE == ELEM_NUM);
@@ -209,27 +214,27 @@ FORCE_INLINE void reduceValueBlock(const float *prob, const scalar_t *v_block,
214
acc[head_elem_idx] = acc[head_elem_idx] + prob_vec * fp32_v_vec;
215
});
216
}
212
-}; // namespace
217
+}; // namespace
218
219
// Paged attention v1
220
namespace {
221
template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE>
222
struct paged_attention_v1_impl {
218
- static void
219
- call(scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
220
- const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
221
- const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
223
+ static void call(
224
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
225
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
226
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
227
// head_size/x, block_size, x]
223
- const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
228
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
229
// head_size, block_size]
225
- const int num_kv_heads, const float scale,
226
- const int
227
- *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
228
- const int *__restrict__ seq_lens, // [num_seqs]
229
- const int max_num_blocks_per_seq,
230
- const float *__restrict__ alibi_slopes, // [num_heads]
231
- const int q_stride, const int kv_block_stride, const int kv_head_stride,
232
- const int num_seqs, const int num_heads) {
230
+ const int num_kv_heads, const float scale,
231
+ const int* __restrict__ block_tables, // [num_seqs,
232
+ // max_num_blocks_per_seq]
233
+ const int* __restrict__ seq_lens, // [num_seqs]
234
+ const int max_num_blocks_per_seq,
235
+ const float* __restrict__ alibi_slopes, // [num_heads]
236
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
237
+ const int num_seqs, const int num_heads) {
238
constexpr int x = 16 / sizeof(scalar_t);
239
const int num_queries_per_kv = num_heads / num_kv_heads;
240
@@ -243,32 +248,31 @@ struct paged_attention_v1_impl {
248
249
size_t logits_bytes =
250
parallel_work_item_num * max_seq_len_padded * sizeof(float);
246
- float *logits = (float *)std::aligned_alloc(
247
- 64, logits_bytes); // Cacheline alignment for each context token.
248
- // [parallel_work_item_num, max_seq_len_padded]
251
+ float* logits = (float*)std::aligned_alloc(
252
+ 64, logits_bytes); // Cacheline alignment for each context token.
253
+ // [parallel_work_item_num, max_seq_len_padded]
254
255
#pragma omp parallel for collapse(2) schedule(dynamic, 1)
256
for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
257
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
258
int seq_len = seq_lens[seq_idx];
254
- const int *seq_block_table =
259
+ const int* seq_block_table =
260
block_tables + max_num_blocks_per_seq * seq_idx;
261
const int block_num = (seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
262
const int64_t kv_head_idx = head_idx / num_queries_per_kv;
258
- const scalar_t *__restrict__ q_vec_ptr =
263
+ const scalar_t* __restrict__ q_vec_ptr =
264
q + seq_idx * q_stride + head_idx * HEAD_SIZE;
260
- const int last_block_token_num =
261
- seq_len - (block_num - 1) * BLOCK_SIZE;
262
- float *__restrict__ thread_block_logits =
265
+ const int last_block_token_num = seq_len - (block_num - 1) * BLOCK_SIZE;
266
+ float* __restrict__ thread_block_logits =
267
logits + omp_get_thread_num() * max_seq_len_padded;
268
269
// Compute logits
270
for (int block_idx = 0; block_idx < block_num; ++block_idx) {
271
const int64_t physical_block_idx = seq_block_table[block_idx];
268
- const scalar_t *__restrict__ k_block_cache_ptr =
272
+ const scalar_t* __restrict__ k_block_cache_ptr =
273
k_cache + physical_block_idx * kv_block_stride +
274
kv_head_idx * kv_head_stride;
271
- float *__restrict__ head_block_logits =
275
+ float* __restrict__ head_block_logits =
276
thread_block_logits + block_idx * BLOCK_SIZE;
277
278
reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
@@ -282,8 +286,7 @@ struct paged_attention_v1_impl {
286
block_num * BLOCK_SIZE, alibi_slopes[head_idx], 0,
287
seq_len);
288
} else {
285
- reduceSoftmax(thread_block_logits, seq_len,
286
- block_num * BLOCK_SIZE);
289
+ reduceSoftmax(thread_block_logits, seq_len, block_num * BLOCK_SIZE);
290
}
291
292
// Compute value
@@ -293,14 +296,14 @@ struct paged_attention_v1_impl {
296
for (int head_part_idx = 0; head_part_idx < head_partition_num;
297
++head_part_idx) {
298
vec_op::FP32Vec16 accums[head_elem_num_per_partition];
296
- scalar_t *__restrict__ out_ptr =
299
+ scalar_t* __restrict__ out_ptr =
300
out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
301
head_part_idx * head_elem_num_per_partition;
302
for (int block_idx = 0; block_idx < block_num; ++block_idx) {
303
const int64_t physical_block_idx = seq_block_table[block_idx];
301
- const float *__restrict__ prob_vec_ptr =
304
+ const float* __restrict__ prob_vec_ptr =
305
thread_block_logits + block_idx * BLOCK_SIZE;
303
- const scalar_t *__restrict__ v_block_cache_ptr =
306
+ const scalar_t* __restrict__ v_block_cache_ptr =
307
v_cache + physical_block_idx * kv_block_stride +
308
kv_head_idx * kv_head_stride +
309
BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -311,7 +314,7 @@ struct paged_attention_v1_impl {
314
if (block_idx != block_num - 1) {
315
const int64_t next_physical_block_idx =
316
seq_block_table[block_idx + 1];
314
- const scalar_t *__restrict__ next_v_block_cache_ptr =
317
+ const scalar_t* __restrict__ next_v_block_cache_ptr =
318
v_cache + next_physical_block_idx * kv_block_stride +
319
kv_head_idx * kv_head_stride +
320
BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -340,16 +343,16 @@ struct paged_attention_v1_impl {
343
#define LAUNCH_V1_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
344
paged_attention_v1_impl<T, HEAD_SIZE, BLOCK_SIZE>::call( \
345
out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
343
- block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
346
+ block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
347
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, num_seqs, \
348
num_heads);
349
350
template <typename T, int BLOCK_SIZE>
351
void paged_attention_v1_impl_launcher(
349
- torch::Tensor &out, torch::Tensor &query, torch::Tensor &key_cache,
350
- torch::Tensor &value_cache, int num_kv_heads, float scale,
351
- torch::Tensor &block_tables, torch::Tensor &seq_lens,
352
- int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
352
+ torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
353
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
354
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
355
+ const c10::optional<torch::Tensor>& alibi_slopes) {
356
int num_seqs = query.size(0);
357
int num_heads = query.size(1);
358
int head_size = query.size(2);
@@ -359,67 +362,66 @@ void paged_attention_v1_impl_launcher(
362
int kv_head_stride = key_cache.stride(1);
363
364
// NOTE: alibi_slopes is optional.
362
- const float *alibi_slopes_ptr =
365
+ const float* alibi_slopes_ptr =
366
alibi_slopes
364
- ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
367
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
368
: nullptr;
369
367
- T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
368
- T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
369
- T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
370
- T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
371
- int *block_tables_ptr = block_tables.data_ptr<int>();
372
- int *seq_lens_ptr = seq_lens.data_ptr<int>();
370
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
371
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
372
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
373
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
374
+ int* block_tables_ptr = block_tables.data_ptr<int>();
375
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
376
377
switch (head_size) {
375
- case 64:
376
- LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
377
- break;
378
- case 80:
379
- LAUNCH_V1_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
380
- break;
381
- case 96:
382
- LAUNCH_V1_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
383
- break;
384
- case 112:
385
- LAUNCH_V1_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
386
- break;
387
- case 128:
388
- LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
389
- break;
390
- case 256:
391
- LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
392
- break;
393
- default:
394
- TORCH_CHECK(false, "Unsupported head size: ", head_size);
395
- break;
378
+ case 64:
379
+ LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
380
+ break;
381
+ case 80:
382
+ LAUNCH_V1_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
383
+ break;
384
+ case 96:
385
+ LAUNCH_V1_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
386
+ break;
387
+ case 112:
388
+ LAUNCH_V1_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
389
+ break;
390
+ case 128:
391
+ LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
392
+ break;
393
+ case 256:
394
+ LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
395
+ break;
396
+ default:
397
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
398
+ break;
399
}
400
}
401
399
-#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
400
- paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
401
- out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
402
+#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
403
+ paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
404
+ out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
405
seq_lens, max_seq_len, alibi_slopes);
406
404
-#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
405
- switch (block_size) { \
406
- case 16: \
407
- CALL_V1_KERNEL_LAUNCHER(T, 16); \
408
- break; \
409
- default: \
410
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
411
- break; \
407
+#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
408
+ switch (block_size) { \
409
+ case 16: \
410
+ CALL_V1_KERNEL_LAUNCHER(T, 16); \
411
+ break; \
412
+ default: \
413
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
414
+ break; \
415
}
413
-} // namespace
416
+} // namespace
417
415
-void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
416
- torch::Tensor &key_cache, torch::Tensor &value_cache,
418
+void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
419
+ torch::Tensor& key_cache, torch::Tensor& value_cache,
420
int num_kv_heads, float scale,
418
- torch::Tensor &block_tables,
419
- torch::Tensor &seq_lens, int block_size,
420
- int max_seq_len,
421
- const c10::optional<torch::Tensor> &alibi_slopes,
422
- const std::string &kv_cache_dtype, float kv_scale) {
421
+ torch::Tensor& block_tables, torch::Tensor& seq_lens,
422
+ int block_size, int max_seq_len,
423
+ const c10::optional<torch::Tensor>& alibi_slopes,
424
+ const std::string& kv_cache_dtype, float kv_scale) {
425
TORCH_CHECK(kv_scale == 1.0f);
426
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
427
[&] {
@@ -434,23 +436,24 @@ namespace {
436
template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE, int PARTITION_SIZE>
437
struct paged_attention_v2_impl {
438
static void call(
437
- scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
438
- float *__restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
439
- float
440
- *__restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
441
- scalar_t *__restrict__ tmp_out, // [num_seqs, num_heads,
442
- // max_num_partitions, head_size]
443
- const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
444
- const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
445
- // head_size/x, block_size, x]
446
- const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
447
- // head_size, block_size]
439
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
440
+ float* __restrict__ exp_sums, // [num_seqs, num_heads,
441
+ // max_num_partitions]
442
+ float* __restrict__ max_logits, // [num_seqs, num_heads,
443
+ // max_num_partitions]
444
+ scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
445
+ // max_num_partitions, head_size]
446
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
447
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
448
+ // head_size/x, block_size, x]
449
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
450
+ // head_size, block_size]
451
const int num_kv_heads, const float scale,
449
- const int
450
- *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
451
- const int *__restrict__ seq_lens, // [num_seqs]
452
+ const int* __restrict__ block_tables, // [num_seqs,
453
+ // max_num_blocks_per_seq]
454
+ const int* __restrict__ seq_lens, // [num_seqs]
455
const int max_num_blocks_per_seq,
453
- const float *__restrict__ alibi_slopes, // [num_heads]
456
+ const float* __restrict__ alibi_slopes, // [num_heads]
457
const int q_stride, const int kv_block_stride, const int kv_head_stride,
458
const int num_seqs, const int num_heads, const int max_num_partitions) {
459
constexpr int x = 16 / sizeof(scalar_t);
@@ -468,8 +471,7 @@ struct paged_attention_v2_impl {
471
const int seq_len = seq_lens[seq_idx];
472
const int start_token_idx = partition_idx * PARTITION_SIZE;
473
471
- if (start_token_idx >= seq_len)
472
- continue;
474
+ if (start_token_idx >= seq_len) continue;
475
476
const int partition_num =
477
(seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
@@ -477,15 +479,14 @@ struct paged_attention_v2_impl {
479
const int token_num =
480
(std::min(seq_len, start_token_idx + PARTITION_SIZE) -
481
start_token_idx);
480
- const int block_num =
481
- (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
482
+ const int block_num = (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
483
const int last_block_token_num =
484
token_num - (block_num - 1) * BLOCK_SIZE;
484
- const int *seq_block_table = block_tables +
485
+ const int* seq_block_table = block_tables +
486
max_num_blocks_per_seq * seq_idx +
487
start_token_idx / BLOCK_SIZE;
488
const int64_t kv_head_idx = head_idx / num_queries_per_kv;
488
- const scalar_t *__restrict__ q_vec_ptr =
489
+ const scalar_t* __restrict__ q_vec_ptr =
490
q + seq_idx * q_stride + head_idx * HEAD_SIZE;
491
492
float logits[PARTITION_SIZE] __attribute__((aligned(64))) = {0};
@@ -493,10 +494,10 @@ struct paged_attention_v2_impl {
494
// Compute logits
495
for (int block_idx = 0; block_idx < block_num; ++block_idx) {
496
const int64_t physical_block_idx = seq_block_table[block_idx];
496
- const scalar_t *__restrict__ k_block_cache_ptr =
497
+ const scalar_t* __restrict__ k_block_cache_ptr =
498
k_cache + physical_block_idx * kv_block_stride +
499
kv_head_idx * kv_head_stride;
499
- float *__restrict__ head_block_logits =
500
+ float* __restrict__ head_block_logits =
501
logits + block_idx * BLOCK_SIZE;
502
503
reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
@@ -510,13 +511,13 @@ struct paged_attention_v2_impl {
511
logits, token_num, block_num * BLOCK_SIZE,
512
alibi_slopes[head_idx], start_token_idx, seq_len);
513
} else {
513
- max_and_sum = reduceSoftmax(logits, token_num,
514
- block_num * BLOCK_SIZE);
514
+ max_and_sum =
515
+ reduceSoftmax(logits, token_num, block_num * BLOCK_SIZE);
516
}
517
517
- auto &&[max_logit, exp_sum] = max_and_sum;
518
+ auto&& [max_logit, exp_sum] = max_and_sum;
519
519
- scalar_t *__restrict__ output_buffer = nullptr;
520
+ scalar_t* __restrict__ output_buffer = nullptr;
521
if (!no_reduce) {
522
auto idx = seq_idx * num_heads * max_num_partitions +
523
head_idx * max_num_partitions + partition_idx;
@@ -538,13 +539,13 @@ struct paged_attention_v2_impl {
539
for (int head_part_idx = 0; head_part_idx < head_partition_num;
540
++head_part_idx) {
541
vec_op::FP32Vec16 accums[head_elem_num_per_partition];
541
- scalar_t *__restrict__ out_ptr =
542
+ scalar_t* __restrict__ out_ptr =
543
output_buffer + head_part_idx * head_elem_num_per_partition;
544
for (int block_idx = 0; block_idx < block_num; ++block_idx) {
545
const int64_t physical_block_idx = seq_block_table[block_idx];
545
- const float *__restrict__ prob_vec_ptr =
546
+ const float* __restrict__ prob_vec_ptr =
547
logits + block_idx * BLOCK_SIZE;
547
- const scalar_t *__restrict__ v_block_cache_ptr =
548
+ const scalar_t* __restrict__ v_block_cache_ptr =
549
v_cache + physical_block_idx * kv_block_stride +
550
kv_head_idx * kv_head_stride +
551
BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -555,7 +556,7 @@ struct paged_attention_v2_impl {
556
if (block_idx != block_num - 1) {
557
const int64_t next_physical_block_idx =
558
seq_block_table[block_idx + 1];
558
- const scalar_t *__restrict__ next_v_block_cache_ptr =
559
+ const scalar_t* __restrict__ next_v_block_cache_ptr =
560
v_cache + next_physical_block_idx * kv_block_stride +
561
kv_head_idx * kv_head_stride +
562
BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
@@ -587,8 +588,7 @@ struct paged_attention_v2_impl {
588
const int partition_num =
589
(seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
590
590
- if (partition_num == 1)
591
- continue;
591
+ if (partition_num == 1) continue;
592
593
reducePartitonSoftmax(
594
max_logits + seq_idx * num_heads * max_num_partitions +
@@ -603,11 +603,11 @@ struct paged_attention_v2_impl {
603
using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
604
static_assert(v_load_vec_type::get_elem_num() == BLOCK_SIZE);
605
constexpr int head_elem_num_per_group =
606
- 16; // Note: didn't align with the cacheline size, due to some HEAD_SIZE
607
- // didn't align with 64 bytes
606
+ 16; // Note: didn't align with the cacheline size, due to some
607
+ // HEAD_SIZE didn't align with 64 bytes
608
static_assert(HEAD_SIZE % head_elem_num_per_group == 0);
609
constexpr int head_group_num = HEAD_SIZE / head_elem_num_per_group;
610
- const float *__restrict__ rescale_factors = exp_sums;
610
+ const float* __restrict__ rescale_factors = exp_sums;
611
#pragma omp parallel for collapse(3) schedule(static, 1)
612
for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
613
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
@@ -616,17 +616,16 @@ struct paged_attention_v2_impl {
616
const int partition_num =
617
(seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
618
619
- if (partition_num == 1)
620
- continue;
619
+ if (partition_num == 1) continue;
620
622
- const float *__restrict__ seq_head_rescale_factors =
621
+ const float* __restrict__ seq_head_rescale_factors =
622
rescale_factors + seq_idx * num_heads * max_num_partitions +
623
head_idx * max_num_partitions;
625
- const scalar_t *__restrict__ seq_head_tmp_out =
624
+ const scalar_t* __restrict__ seq_head_tmp_out =
625
tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
626
head_idx * max_num_partitions * HEAD_SIZE +
627
group_idx * head_elem_num_per_group;
629
- scalar_t *__restrict__ seq_head_output =
628
+ scalar_t* __restrict__ seq_head_output =
629
out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
630
group_idx * head_elem_num_per_group;
631
@@ -645,21 +644,21 @@ struct paged_attention_v2_impl {
644
}
645
};
646
648
-#define LAUNCH_V2_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
649
- paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
650
- out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
651
- key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
652
- seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
653
- kv_block_stride, kv_head_stride, num_seqs, num_heads, \
647
+#define LAUNCH_V2_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
648
+ paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
649
+ out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
650
+ key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
651
+ seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
652
+ kv_block_stride, kv_head_stride, num_seqs, num_heads, \
653
max_num_partitions);
654
655
template <typename T, int BLOCK_SIZE, int PARTITION_SIZE = 512>
656
void paged_attention_v2_impl_launcher(
658
- torch::Tensor &out, torch::Tensor &exp_sums, torch::Tensor &max_logits,
659
- torch::Tensor &tmp_out, torch::Tensor &query, torch::Tensor &key_cache,
660
- torch::Tensor &value_cache, int num_kv_heads, float scale,
661
- torch::Tensor &block_tables, torch::Tensor &seq_lens, int block_size,
662
- int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
657
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
658
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
659
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
660
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
661
+ int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes) {
662
int num_seqs = query.size(0);
663
int num_heads = query.size(1);
664
int head_size = query.size(2);
@@ -670,72 +669,72 @@ void paged_attention_v2_impl_launcher(
669
int max_num_partitions = exp_sums.size(-1);
670
671
// NOTE: alibi_slopes is optional.
673
- const float *alibi_slopes_ptr =
672
+ const float* alibi_slopes_ptr =
673
alibi_slopes
675
- ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
674
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
675
: nullptr;
676
678
- T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
679
- float *exp_sums_ptr = reinterpret_cast<float *>(exp_sums.data_ptr());
680
- float *max_logits_ptr = reinterpret_cast<float *>(max_logits.data_ptr());
681
- T *tmp_out_ptr = reinterpret_cast<T *>(tmp_out.data_ptr());
682
- T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
683
- T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
684
- T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
685
- int *block_tables_ptr = block_tables.data_ptr<int>();
686
- int *seq_lens_ptr = seq_lens.data_ptr<int>();
677
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
678
+ float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
679
+ float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
680
+ T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
681
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
682
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
683
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
684
+ int* block_tables_ptr = block_tables.data_ptr<int>();
685
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
686
687
switch (head_size) {
689
- case 64:
690
- LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
691
- break;
692
- case 80:
693
- LAUNCH_V2_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
694
- break;
695
- case 96:
696
- LAUNCH_V2_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
697
- break;
698
- case 112:
699
- LAUNCH_V2_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
700
- break;
701
- case 128:
702
- LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
703
- break;
704
- case 256:
705
- LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
706
- break;
707
- default:
708
- TORCH_CHECK(false, "Unsupported head size: ", head_size);
709
- break;
688
+ case 64:
689
+ LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
690
+ break;
691
+ case 80:
692
+ LAUNCH_V2_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
693
+ break;
694
+ case 96:
695
+ LAUNCH_V2_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
696
+ break;
697
+ case 112:
698
+ LAUNCH_V2_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
699
+ break;
700
+ case 128:
701
+ LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
702
+ break;
703
+ case 256:
704
+ LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
705
+ break;
706
+ default:
707
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
708
+ break;
709
}
710
}
711
713
-#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
714
- paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
715
- out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
716
- num_kv_heads, scale, block_tables, seq_lens, block_size, \
717
- max_seq_len, alibi_slopes);
718
-
719
-#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
720
- switch (block_size) { \
721
- case 16: \
722
- CALL_V2_KERNEL_LAUNCHER(T, 16); \
723
- break; \
724
- default: \
725
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
726
- break; \
712
+#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
713
+ paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
714
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
715
+ num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len, \
716
+ alibi_slopes);
717
+
718
+#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
719
+ switch (block_size) { \
720
+ case 16: \
721
+ CALL_V2_KERNEL_LAUNCHER(T, 16); \
722
+ break; \
723
+ default: \
724
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
725
+ break; \
726
}
728
-} // namespace
729
-
730
-void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
731
- torch::Tensor &max_logits, torch::Tensor &tmp_out,
732
- torch::Tensor &query, torch::Tensor &key_cache,
733
- torch::Tensor &value_cache, int num_kv_heads,
734
- float scale, torch::Tensor &block_tables,
735
- torch::Tensor &seq_lens, int block_size,
727
+} // namespace
728
+
729
+void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
730
+ torch::Tensor& max_logits, torch::Tensor& tmp_out,
731
+ torch::Tensor& query, torch::Tensor& key_cache,
732
+ torch::Tensor& value_cache, int num_kv_heads,
733
+ float scale, torch::Tensor& block_tables,
734
+ torch::Tensor& seq_lens, int block_size,
735
int max_seq_len,
737
- const c10::optional<torch::Tensor> &alibi_slopes,
738
- const std::string &kv_cache_dtype, float kv_scale) {
736
+ const c10::optional<torch::Tensor>& alibi_slopes,
737
+ const std::string& kv_cache_dtype, float kv_scale) {
738
TORCH_CHECK(kv_scale == 1.0f);
739
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
740
[&] {
csrc/cpu/cache.cpp
+27
-26
@@ -5,25 +5,26 @@
5
6
namespace {
7
template <typename scalar_t>
8
-void copy_blocks_cpu_impl(
9
- std::vector<torch::Tensor> &key_caches,
10
- std::vector<torch::Tensor> &value_caches,
11
- const torch::Tensor& mapping_pairs,
12
- const int element_num_per_block, const int layer_num) {
8
+void copy_blocks_cpu_impl(std::vector<torch::Tensor>& key_caches,
9
+ std::vector<torch::Tensor>& value_caches,
10
+ const torch::Tensor& mapping_pairs,
11
+ const int element_num_per_block,
12
+ const int layer_num) {
13
const size_t pair_num = mapping_pairs.size(0);
14
const size_t block_bytes = sizeof(scalar_t) * element_num_per_block;
15
#pragma omp parallel for collapse(2)
16
for (int layer = 0; layer < layer_num; ++layer) {
17
for (size_t pair = 0; pair < pair_num; ++pair) {
18
- int64_t source_offset = element_num_per_block * mapping_pairs[pair][0].item<int64_t>();
18
+ int64_t source_offset =
19
+ element_num_per_block * mapping_pairs[pair][0].item<int64_t>();
20
int64_t target_offset =
21
element_num_per_block * mapping_pairs[pair][1].item<int64_t>();
21
- scalar_t *key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
22
- scalar_t *source_ptr = key_cache_ptr + source_offset;
23
- scalar_t *target_ptr = key_cache_ptr + target_offset;
22
+ scalar_t* key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
23
+ scalar_t* source_ptr = key_cache_ptr + source_offset;
24
+ scalar_t* target_ptr = key_cache_ptr + target_offset;
25
std::memcpy(target_ptr, source_ptr, block_bytes);
26
26
- scalar_t *value_cache_ptr = value_caches[layer].data_ptr<scalar_t>();
27
+ scalar_t* value_cache_ptr = value_caches[layer].data_ptr<scalar_t>();
28
source_ptr = value_cache_ptr + source_offset;
29
target_ptr = value_cache_ptr + target_offset;
30
std::memcpy(target_ptr, source_ptr, block_bytes);
@@ -33,9 +34,9 @@ void copy_blocks_cpu_impl(
34
35
template <typename scalar_t>
36
void reshape_and_cache_cpu_impl(
36
- const scalar_t *__restrict__ key, const scalar_t *__restrict__ value,
37
- scalar_t *__restrict__ key_cache, scalar_t *__restrict__ value_cache,
38
- const int64_t *__restrict__ slot_mapping, const int num_tokens,
37
+ const scalar_t* __restrict__ key, const scalar_t* __restrict__ value,
38
+ scalar_t* __restrict__ key_cache, scalar_t* __restrict__ value_cache,
39
+ const int64_t* __restrict__ slot_mapping, const int num_tokens,
40
const int key_stride, const int value_stride, const int num_heads,
41
const int head_size, const int block_size, const int x) {
42
const int block_elem_num = num_heads * head_size * block_size;
@@ -48,14 +49,14 @@ void reshape_and_cache_cpu_impl(
49
int src_key_head_idx = token_idx * key_stride + head_idx * head_size;
50
int src_value_head_idx =
51
token_idx * value_stride + head_idx * head_size;
51
- const scalar_t *src_key_head_ptr = key + src_key_head_idx;
52
- const scalar_t *src_value_head_ptr = value + src_value_head_idx;
52
+ const scalar_t* src_key_head_ptr = key + src_key_head_idx;
53
+ const scalar_t* src_value_head_ptr = value + src_value_head_idx;
54
const int64_t block_index = slot_idx / block_size;
55
const int64_t block_offset = slot_idx % block_size;
55
- scalar_t *target_key_head_ptr = key_cache +
56
+ scalar_t* target_key_head_ptr = key_cache +
57
block_elem_num * block_index +
58
head_idx * block_size * head_size;
58
- scalar_t *target_value_head_ptr = value_cache +
59
+ scalar_t* target_value_head_ptr = value_cache +
60
block_elem_num * block_index +
61
head_idx * block_size * head_size;
62
@@ -79,10 +80,10 @@ void reshape_and_cache_cpu_impl(
80
}
81
}
82
}
82
-}; // namespace
83
+}; // namespace
84
84
-void copy_blocks(std::vector<torch::Tensor> &key_caches,
85
- std::vector<torch::Tensor> &value_caches,
85
+void copy_blocks(std::vector<torch::Tensor>& key_caches,
86
+ std::vector<torch::Tensor>& value_caches,
87
const torch::Tensor& block_mapping) {
88
unsigned num_layers = key_caches.size();
89
TORCH_CHECK(num_layers == value_caches.size());
@@ -100,10 +101,10 @@ void copy_blocks(std::vector<torch::Tensor> &key_caches,
101
});
102
}
103
103
-void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
104
- torch::Tensor &key_cache, torch::Tensor &value_cache,
105
- torch::Tensor &slot_mapping,
106
- const std::string &kv_cache_dtype, float kv_scale) {
104
+void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
105
+ torch::Tensor& key_cache, torch::Tensor& value_cache,
106
+ torch::Tensor& slot_mapping,
107
+ const std::string& kv_cache_dtype, float kv_scale) {
108
TORCH_CHECK(kv_scale == 1.0f);
109
110
int num_tokens = key.size(0);
@@ -127,7 +128,7 @@ void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
128
});
129
}
130
130
-void swap_blocks(torch::Tensor &src, torch::Tensor &dst,
131
- const torch::Tensor&block_mapping) {
131
+void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
132
+ const torch::Tensor& block_mapping) {
133
TORCH_CHECK(false, "swap_blocks is unsupported on CPU.")
134
}
csrc/cpu/layernorm.cpp
+16
-16
@@ -2,10 +2,10 @@
2
3
namespace {
4
template <typename scalar_t>
5
-void rms_norm_impl(scalar_t *__restrict__ out,
6
- const scalar_t *__restrict__ input,
7
- const scalar_t *__restrict__ weight, const float epsilon,
8
- const int num_tokens, const int hidden_size) {
5
+void rms_norm_impl(scalar_t* __restrict__ out,
6
+ const scalar_t* __restrict__ input,
7
+ const scalar_t* __restrict__ weight, const float epsilon,
8
+ const int num_tokens, const int hidden_size) {
9
using scalar_vec_t = vec_op::vec_t<scalar_t>;
10
constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
11
TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -41,11 +41,11 @@ void rms_norm_impl(scalar_t *__restrict__ out,
41
}
42
43
template <typename scalar_t>
44
-void fused_add_rms_norm_impl(scalar_t *__restrict__ input,
45
- scalar_t *__restrict__ residual,
46
- const scalar_t *__restrict__ weight,
47
- const float epsilon, const int num_tokens,
48
- const int hidden_size) {
44
+void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
45
+ scalar_t* __restrict__ residual,
46
+ const scalar_t* __restrict__ weight,
47
+ const float epsilon, const int num_tokens,
48
+ const int hidden_size) {
49
using scalar_vec_t = vec_op::vec_t<scalar_t>;
50
constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
51
TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -85,24 +85,24 @@ void fused_add_rms_norm_impl(scalar_t *__restrict__ input,
85
}
86
}
87
}
88
-} // namespace
88
+} // namespace
89
90
-void rms_norm(torch::Tensor &out, torch::Tensor &input,
91
- torch::Tensor &weight, float epsilon) {
90
+void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
91
+ float epsilon) {
92
int hidden_size = input.size(-1);
93
int num_tokens = input.numel() / hidden_size;
94
95
VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_impl", [&] {
96
CPU_KERNEL_GUARD_IN(rms_norm_impl)
97
rms_norm_impl(out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
98
- weight.data_ptr<scalar_t>(), epsilon, num_tokens,
99
- hidden_size);
98
+ weight.data_ptr<scalar_t>(), epsilon, num_tokens,
99
+ hidden_size);
100
CPU_KERNEL_GUARD_OUT(rms_norm_impl)
101
});
102
}
103
104
-void fused_add_rms_norm(torch::Tensor &input, torch::Tensor &residual,
105
- torch::Tensor &weight, float epsilon) {
104
+void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
105
+ torch::Tensor& weight, float epsilon) {
106
int hidden_size = input.size(-1);
107
int num_tokens = input.numel() / hidden_size;
108
csrc/cpu/pos_encoding.cpp
+33
-33
@@ -4,16 +4,16 @@
4
namespace {
5
template <typename scalar_t>
6
void rotary_embedding_impl(
7
- const int64_t
8
- *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
9
- scalar_t
10
- *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
11
- /// [num_tokens, num_heads, head_size]
12
- scalar_t
13
- *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
14
- // [num_tokens, num_kv_heads, head_size]
15
- const scalar_t
16
- *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
7
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or
8
+ // [num_tokens]
9
+ scalar_t* __restrict__ query, /// [batch_size, seq_len, num_heads,
10
+ /// head_size] or [num_tokens, num_heads,
11
+ /// head_size]
12
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
13
+ // head_size] or [num_tokens, num_kv_heads,
14
+ // head_size]
15
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
16
+ // 2]
17
const int rot_dim, const int64_t query_stride, const int64_t key_stride,
18
const int num_heads, const int num_kv_heads, const int head_size,
19
const int num_tokens) {
@@ -26,7 +26,7 @@ void rotary_embedding_impl(
26
#pragma omp parallel for
27
for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
28
int64_t pos = positions[token_idx];
29
- const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
29
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
30
31
for (int i = 0; i < num_heads; ++i) {
32
const int head_idx = i;
@@ -94,16 +94,16 @@ void rotary_embedding_impl(
94
95
template <typename scalar_t>
96
void rotary_embedding_gptj_impl(
97
- const int64_t
98
- *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99
- scalar_t
100
- *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
101
- /// [num_tokens, num_heads, head_size]
102
- scalar_t
103
- *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
104
- // [num_tokens, num_kv_heads, head_size]
105
- const scalar_t
106
- *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
97
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or
98
+ // [num_tokens]
99
+ scalar_t* __restrict__ query, /// [batch_size, seq_len, num_heads,
100
+ /// head_size] or [num_tokens, num_heads,
101
+ /// head_size]
102
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
103
+ // head_size] or [num_tokens, num_kv_heads,
104
+ // head_size]
105
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
106
+ // 2]
107
const int rot_dim, const int64_t query_stride, const int64_t key_stride,
108
const int num_heads, const int num_kv_heads, const int head_size,
109
const int num_tokens) {
@@ -113,13 +113,13 @@ void rotary_embedding_gptj_impl(
113
for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
114
for (int i = 0; i < num_heads; ++i) {
115
int64_t pos = positions[token_idx];
116
- const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
117
- const scalar_t *cos_cache_ptr = cache_ptr;
118
- const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
116
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
117
+ const scalar_t* cos_cache_ptr = cache_ptr;
118
+ const scalar_t* sin_cache_ptr = cache_ptr + embed_dim;
119
const int head_idx = i;
120
const int64_t token_head =
121
token_idx * query_stride + head_idx * head_size;
122
- scalar_t *head_query = token_head + query;
122
+ scalar_t* head_query = token_head + query;
123
for (int j = 0; j < embed_dim; j += 1) {
124
const int rot_offset = j;
125
const int x_index = 2 * rot_offset;
@@ -141,12 +141,12 @@ void rotary_embedding_gptj_impl(
141
for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
142
for (int i = 0; i < num_kv_heads; ++i) {
143
int64_t pos = positions[token_idx];
144
- const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
145
- const scalar_t *cos_cache_ptr = cache_ptr;
146
- const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
144
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
145
+ const scalar_t* cos_cache_ptr = cache_ptr;
146
+ const scalar_t* sin_cache_ptr = cache_ptr + embed_dim;
147
const int head_idx = i;
148
const int64_t token_head = token_idx * key_stride + head_idx * head_size;
149
- scalar_t *head_key = key + token_head;
149
+ scalar_t* head_key = key + token_head;
150
for (int j = 0; j < embed_dim; j += 1) {
151
const int rot_offset = j;
152
const int x_index = 2 * rot_offset;
@@ -164,11 +164,11 @@ void rotary_embedding_gptj_impl(
164
}
165
}
166
}
167
-}; // namespace
167
+}; // namespace
168
169
-void rotary_embedding(torch::Tensor &positions, torch::Tensor &query,
170
- torch::Tensor &key, int head_size,
171
- torch::Tensor &cos_sin_cache, bool is_neox) {
169
+void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
170
+ torch::Tensor& key, int head_size,
171
+ torch::Tensor& cos_sin_cache, bool is_neox) {
172
int num_tokens = query.numel() / query.size(-1);
173
int rot_dim = cos_sin_cache.size(1);
174
int num_heads = query.size(-1) / head_size;
csrc/cpu/pybind.cpp
+23
-52
@@ -8,66 +8,37 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8
pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
10
// Attention ops
11
- ops.def(
12
- "paged_attention_v1",
13
- &paged_attention_v1,
14
- "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15
- ops.def(
16
- "paged_attention_v2",
17
- &paged_attention_v2,
18
- "PagedAttention V2.");
11
+ ops.def("paged_attention_v1", &paged_attention_v1,
12
+ "Compute the attention between an input query and the cached "
13
+ "keys/values using PagedAttention.");
14
+ ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15
16
// Activation ops
21
- ops.def(
22
- "silu_and_mul",
23
- &silu_and_mul,
24
- "Activation function used in SwiGLU.");
25
- ops.def(
26
- "gelu_and_mul",
27
- &gelu_and_mul,
28
- "Activation function used in GeGLU with `none` approximation.");
29
- ops.def(
30
- "gelu_tanh_and_mul",
31
- &gelu_tanh_and_mul,
32
- "Activation function used in GeGLU with `tanh` approximation.");
33
- ops.def(
34
- "gelu_new",
35
- &gelu_new,
36
- "GELU implementation used in GPT-2.");
37
- ops.def(
38
- "gelu_fast",
39
- &gelu_fast,
40
- "Approximate GELU implementation.");
17
+ ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18
+ ops.def("gelu_and_mul", &gelu_and_mul,
19
+ "Activation function used in GeGLU with `none` approximation.");
20
+ ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21
+ "Activation function used in GeGLU with `tanh` approximation.");
22
+ ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23
+ ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24
25
// Layernorm
43
- ops.def(
44
- "rms_norm",
45
- &rms_norm,
46
- "Apply Root Mean Square (RMS) Normalization to the input tensor.");
26
+ ops.def("rms_norm", &rms_norm,
27
+ "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28
48
- ops.def(
49
- "fused_add_rms_norm",
50
- &fused_add_rms_norm,
51
- "In-place fused Add and RMS Normalization");
29
+ ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30
+ "In-place fused Add and RMS Normalization");
31
32
// Rotary embedding
54
- ops.def(
55
- "rotary_embedding",
56
- &rotary_embedding,
57
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
33
+ ops.def("rotary_embedding", &rotary_embedding,
34
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35
36
// Cache ops
37
pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
61
- cache_ops.def(
62
- "swap_blocks",
63
- &swap_blocks,
64
- "Swap in (out) the cache blocks from src to dst");
65
- cache_ops.def(
66
- "copy_blocks",
67
- ©_blocks,
68
- "Copy the cache blocks from src to dst");
69
- cache_ops.def(
70
- "reshape_and_cache",
71
- &reshape_and_cache,
72
- "Reshape the key and value tensors and cache them");
38
+ cache_ops.def("swap_blocks", &swap_blocks,
39
+ "Swap in (out) the cache blocks from src to dst");
40
+ cache_ops.def("copy_blocks", ©_blocks,
41
+ "Copy the cache blocks from src to dst");
42
+ cache_ops.def("reshape_and_cache", &reshape_and_cache,
43
+ "Reshape the key and value tensors and cache them");
44
}
csrc/cuda_compat.h
+5
-4
@@ -1,7 +1,7 @@
1
#pragma once
2
3
#ifdef USE_ROCM
4
-#include <hip/hip_runtime.h>
4
+ #include <hip/hip_runtime.h>
5
#endif
6
7
#ifndef USE_ROCM
@@ -17,7 +17,8 @@
17
#endif
18
19
#ifndef USE_ROCM
20
- #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor_sync(uint32_t(-1), var, lane_mask)
20
+ #define VLLM_SHFL_XOR_SYNC(var, lane_mask) \
21
+ __shfl_xor_sync(uint32_t(-1), var, lane_mask)
22
#else
23
#define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
24
#endif
@@ -29,7 +30,8 @@
30
#endif
31
32
#ifndef USE_ROCM
32
- #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down_sync(uint32_t(-1), var, lane_delta)
33
+ #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) \
34
+ __shfl_down_sync(uint32_t(-1), var, lane_delta)
35
#else
36
#define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down(var, lane_delta)
37
#endif
@@ -41,4 +43,3 @@
43
#define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
44
hipFuncSetAttribute(FUNC, hipFuncAttributeMaxDynamicSharedMemorySize, VAL)
45
#endif
44
-
csrc/cuda_utils.h
+2
-5
@@ -2,9 +2,6 @@
2
3
#include <torch/extension.h>
4
5
-int get_device_attribute(
6
- int attribute,
7
- int device_id);
5
+int get_device_attribute(int attribute, int device_id);
6
9
-int get_max_shared_memory_per_block_device_attribute(
10
- int device_id);
7
+int get_max_shared_memory_per_block_device_attribute(int device_id);
csrc/cuda_utils_kernels.cu
+17
-23
@@ -2,34 +2,28 @@
2
#include <hip/hip_runtime.h>
3
#include <hip/hip_runtime_api.h>
4
#endif
5
-int get_device_attribute(
6
- int attribute,
7
- int device_id)
8
-{
9
- int device, value;
10
- if (device_id < 0) {
11
- cudaGetDevice(&device);
12
- }
13
- else {
14
- device = device_id;
15
- }
16
- cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute), device);
17
- return value;
5
+int get_device_attribute(int attribute, int device_id) {
6
+ int device, value;
7
+ if (device_id < 0) {
8
+ cudaGetDevice(&device);
9
+ } else {
10
+ device = device_id;
11
+ }
12
+ cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute),
13
+ device);
14
+ return value;
15
}
16
20
-
21
-int get_max_shared_memory_per_block_device_attribute(
22
- int device_id)
23
-{
24
-int attribute;
25
-// https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
26
-// cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
17
+int get_max_shared_memory_per_block_device_attribute(int device_id) {
18
+ int attribute;
19
+ // https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
20
+ // cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
21
22
#ifdef USE_ROCM
29
- attribute = hipDeviceAttributeMaxSharedMemoryPerBlock;
23
+ attribute = hipDeviceAttributeMaxSharedMemoryPerBlock;
24
#else
31
- attribute = cudaDevAttrMaxSharedMemoryPerBlockOptin;
25
+ attribute = cudaDevAttrMaxSharedMemoryPerBlockOptin;
26
#endif
27
34
- return get_device_attribute(attribute, device_id);
28
+ return get_device_attribute(attribute, device_id);
29
}
csrc/custom_all_reduce.cu
+27
-28
@@ -7,11 +7,11 @@
7
8
// fake pointer type
9
using fptr_t = uint64_t;
10
-static_assert(sizeof(void *) == sizeof(fptr_t));
10
+static_assert(sizeof(void*) == sizeof(fptr_t));
11
12
-fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
13
- const std::vector<std::string> &handles,
14
- const std::vector<int64_t> &offsets, int rank,
12
+fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
13
+ const std::vector<std::string>& handles,
14
+ const std::vector<int64_t>& offsets, int rank,
15
bool full_nvlink) {
16
int world_size = offsets.size();
17
if (world_size > 8)
@@ -29,7 +29,7 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
29
std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(cudaIpcMemHandle_t));
30
}
31
return (fptr_t) new vllm::CustomAllreduce(
32
- reinterpret_cast<vllm::Signal *>(meta.data_ptr()), rank_data.data_ptr(),
32
+ reinterpret_cast<vllm::Signal*>(meta.data_ptr()), rank_data.data_ptr(),
33
rank_data.numel(), ipc_handles, offsets, rank, full_nvlink);
34
}
35
@@ -49,13 +49,13 @@ fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
49
* 5. A[None].expand(2, -1, -1, -1): Not OK
50
* 6. A[:, 1:, 1:]: Not OK
51
*/
52
-bool _is_weak_contiguous(torch::Tensor &t) {
52
+bool _is_weak_contiguous(torch::Tensor& t) {
53
return t.is_contiguous() ||
54
(t.storage().nbytes() - t.storage_offset() * t.element_size() ==
55
t.numel() * t.element_size());
56
}
57
58
-bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
58
+bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
59
bool full_nvlink) {
60
auto inp_size = inp.numel() * inp.element_size();
61
// custom allreduce requires input byte size to be multiples of 16
@@ -67,28 +67,27 @@ bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
67
return false;
68
}
69
70
-void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
70
+void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
71
cudaStream_t stream) {
72
- auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
72
+ auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
73
TORCH_CHECK(_is_weak_contiguous(out));
74
switch (out.scalar_type()) {
75
case at::ScalarType::Float: {
76
- fa->allreduce<float>(stream, reinterpret_cast<float *>(inp.data_ptr()),
77
- reinterpret_cast<float *>(out.data_ptr()),
76
+ fa->allreduce<float>(stream, reinterpret_cast<float*>(inp.data_ptr()),
77
+ reinterpret_cast<float*>(out.data_ptr()),
78
out.numel());
79
break;
80
}
81
case at::ScalarType::Half: {
82
- fa->allreduce<half>(stream, reinterpret_cast<half *>(inp.data_ptr()),
83
- reinterpret_cast<half *>(out.data_ptr()),
84
- out.numel());
82
+ fa->allreduce<half>(stream, reinterpret_cast<half*>(inp.data_ptr()),
83
+ reinterpret_cast<half*>(out.data_ptr()), out.numel());
84
break;
85
}
86
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
87
case at::ScalarType::BFloat16: {
88
fa->allreduce<nv_bfloat16>(
90
- stream, reinterpret_cast<nv_bfloat16 *>(inp.data_ptr()),
91
- reinterpret_cast<nv_bfloat16 *>(out.data_ptr()), out.numel());
89
+ stream, reinterpret_cast<nv_bfloat16*>(inp.data_ptr()),
90
+ reinterpret_cast<nv_bfloat16*>(out.data_ptr()), out.numel());
91
break;
92
}
93
#endif
@@ -98,7 +97,7 @@ void _all_reduce(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out,
97
}
98
}
99
101
-void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
100
+void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) {
101
const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
102
auto stream = c10::cuda::getCurrentCUDAStream().stream();
103
TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
@@ -106,8 +105,8 @@ void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out) {
105
_all_reduce(_fa, inp, out, stream);
106
}
107
109
-void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor ®_buffer,
110
- torch::Tensor &out) {
108
+void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
109
+ torch::Tensor& out) {
110
const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
111
auto stream = c10::cuda::getCurrentCUDAStream().stream();
112
@@ -122,27 +121,27 @@ void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor ®_buffer,
121
}
122
123
void dispose(fptr_t _fa) {
125
- auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
124
+ auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
125
delete fa;
126
}
127
128
int meta_size() { return sizeof(vllm::Signal); }
129
131
-void register_buffer(fptr_t _fa, torch::Tensor &t,
132
- const std::vector<std::string> &handles,
133
- const std::vector<int64_t> &offsets) {
134
- auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
130
+void register_buffer(fptr_t _fa, torch::Tensor& t,
131
+ const std::vector<std::string>& handles,
132
+ const std::vector<int64_t>& offsets) {
133
+ auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
134
fa->register_buffer(handles, offsets, t.data_ptr());
135
}
136
137
std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
138
fptr_t _fa) {
140
- auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
139
+ auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
140
return fa->get_graph_buffer_ipc_meta();
141
}
142
144
-void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
145
- const std::vector<std::vector<int64_t>> &offsets) {
146
- auto fa = reinterpret_cast<vllm::CustomAllreduce *>(_fa);
143
+void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
144
+ const std::vector<std::vector<int64_t>>& offsets) {
145
+ auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
146
fa->register_graph_buffers(handles, offsets);
147
}
csrc/custom_all_reduce.cuh
+51
-54
@@ -31,9 +31,9 @@ struct Signal {
31
alignas(128) uint32_t end[kMaxBlocks][8];
32
};
33
34
-struct __align__(16) RankData { const void *__restrict__ ptrs[8]; };
34
+struct __align__(16) RankData { const void* __restrict__ ptrs[8]; };
35
36
-struct __align__(16) RankSignals { volatile Signal *signals[8]; };
36
+struct __align__(16) RankSignals { volatile Signal* signals[8]; };
37
38
// like std::array, but aligned
39
template <typename T, int sz>
@@ -68,11 +68,11 @@ DINLINE half downcast_s(float val) {
68
// scalar add functions
69
// for some reason when compiling with Pytorch, the + operator for half and
70
// bfloat is disabled so we call the intrinsics directly
71
-DINLINE half &assign_add(half &a, half b) {
71
+DINLINE half& assign_add(half& a, half b) {
72
a = __hadd(a, b);
73
return a;
74
}
75
-DINLINE float &assign_add(float &a, float b) { return a += b; }
75
+DINLINE float& assign_add(float& a, float b) { return a += b; }
76
77
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
78
DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
@@ -80,14 +80,14 @@ template <>
80
DINLINE nv_bfloat16 downcast_s(float val) {
81
return __float2bfloat16(val);
82
}
83
-DINLINE nv_bfloat16 &assign_add(nv_bfloat16 &a, nv_bfloat16 b) {
83
+DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) {
84
a = __hadd(a, b);
85
return a;
86
}
87
#endif
88
89
template <typename T, int N>
90
-DINLINE array_t<T, N> &packed_assign_add(array_t<T, N> &a, array_t<T, N> b) {
90
+DINLINE array_t<T, N>& packed_assign_add(array_t<T, N>& a, array_t<T, N> b) {
91
#pragma unroll
92
for (int i = 0; i < N; i++) {
93
assign_add(a.data[i], b.data[i]);
@@ -128,7 +128,7 @@ DINLINE O downcast(array_t<float, O::size> val) {
128
// prior memory accesses. Note: volatile writes will not be reordered against
129
// other volatile writes.
130
template <int ngpus>
131
-DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
131
+DINLINE void start_sync(const RankSignals& sg, volatile Signal* self_sg,
132
int rank) {
133
if (threadIdx.x < ngpus) {
134
// reset flag for next time
@@ -137,8 +137,7 @@ DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
137
// Latency = 1 p2p write
138
sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1;
139
// wait until we got true from all ranks
140
- while (!self_sg->start[blockIdx.x][threadIdx.x])
141
- ;
140
+ while (!self_sg->start[blockIdx.x][threadIdx.x]);
141
}
142
__syncthreads();
143
}
@@ -147,13 +146,13 @@ DINLINE void start_sync(const RankSignals &sg, volatile Signal *self_sg,
146
// barrier in the all reduce kernel. If it's the final synchronization barrier,
147
// we don't need to make any visibility guarantees for prior memory accesses.
148
template <int ngpus, bool final_sync = false>
150
-DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
149
+DINLINE void end_sync(const RankSignals& sg, volatile Signal* self_sg,
150
int rank) {
151
__syncthreads();
152
// eliminate the case that prior writes are not visible after signals become
153
// visible. Note that I did not managed to make this happen through a lot of
154
// testing. Might be the case that hardware provides stronger guarantee than
156
- // the memory model.
155
+ // the memory model.
156
if constexpr (!final_sync) __threadfence_system();
157
if (threadIdx.x < ngpus) {
158
// reset flag for next time
@@ -162,14 +161,13 @@ DINLINE void end_sync(const RankSignals &sg, volatile Signal *self_sg,
161
// Latency = 1 p2p write
162
sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1;
163
// wait until we got true from all ranks
165
- while (!self_sg->end[blockIdx.x][threadIdx.x])
166
- ;
164
+ while (!self_sg->end[blockIdx.x][threadIdx.x]);
165
}
166
if constexpr (!final_sync) __syncthreads();
167
}
168
169
template <typename P, int ngpus, typename A>
172
-DINLINE P packed_reduce(const P *ptrs[], int idx) {
170
+DINLINE P packed_reduce(const P* ptrs[], int idx) {
171
A tmp = upcast(ptrs[0][idx]);
172
#pragma unroll
173
for (int i = 1; i < ngpus; i++) {
@@ -180,8 +178,8 @@ DINLINE P packed_reduce(const P *ptrs[], int idx) {
178
179
template <typename T, int ngpus>
180
__global__ void __launch_bounds__(512, 1)
183
- cross_device_reduce_1stage(RankData *_dp, RankSignals sg,
184
- volatile Signal *self_sg, T *__restrict__ result,
181
+ cross_device_reduce_1stage(RankData* _dp, RankSignals sg,
182
+ volatile Signal* self_sg, T* __restrict__ result,
183
int rank, int size) {
184
using P = typename packed_t<T>::P;
185
using A = typename packed_t<T>::A;
@@ -192,21 +190,20 @@ __global__ void __launch_bounds__(512, 1)
190
// do the actual reduction
191
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
192
idx += gridDim.x * blockDim.x) {
195
- ((P *)result)[idx] =
196
- packed_reduce<P, ngpus, A>((const P **)&dp.ptrs[0], idx);
193
+ ((P*)result)[idx] = packed_reduce<P, ngpus, A>((const P**)&dp.ptrs[0], idx);
194
}
195
end_sync<ngpus, true>(sg, self_sg, rank);
196
}
197
198
template <typename P>
202
-DINLINE P *get_tmp_buf(volatile Signal *sg) {
203
- return (P *)(((Signal *)sg) + 1);
199
+DINLINE P* get_tmp_buf(volatile Signal* sg) {
200
+ return (P*)(((Signal*)sg) + 1);
201
}
202
203
template <typename T, int ngpus>
204
__global__ void __launch_bounds__(512, 1)
208
- cross_device_reduce_2stage(RankData *_dp, RankSignals sg,
209
- volatile Signal *self_sg, T *__restrict__ result,
205
+ cross_device_reduce_2stage(RankData* _dp, RankSignals sg,
206
+ volatile Signal* self_sg, T* __restrict__ result,
207
int rank, int size) {
208
int tid = blockIdx.x * blockDim.x + threadIdx.x;
209
int stride = gridDim.x * blockDim.x;
@@ -216,12 +213,12 @@ __global__ void __launch_bounds__(512, 1)
213
int start = rank * part;
214
int end = rank == ngpus - 1 ? size : start + part;
215
int largest_part = part + size % ngpus;
219
- const P *ptrs[ngpus];
220
- P *tmps[ngpus];
216
+ const P* ptrs[ngpus];
217
+ P* tmps[ngpus];
218
#pragma unroll
219
for (int i = 0; i < ngpus; i++) {
220
int target = (rank + i) % ngpus;
224
- ptrs[i] = (const P *)_dp->ptrs[target];
221
+ ptrs[i] = (const P*)_dp->ptrs[target];
222
tmps[i] = get_tmp_buf<P>(sg.signals[target]);
223
}
224
auto tmp_out = tmps[0];
@@ -243,7 +240,7 @@ __global__ void __launch_bounds__(512, 1)
240
int gather_from_rank = ((rank + i) % ngpus);
241
if (gather_from_rank == ngpus - 1 || idx < part) {
242
int dst_idx = gather_from_rank * part + idx;
246
- ((P *)result)[dst_idx] = tmps[i][idx];
243
+ ((P*)result)[dst_idx] = tmps[i][idx];
244
}
245
}
246
}
@@ -261,14 +258,14 @@ class CustomAllreduce {
258
259
// below are device pointers
260
RankSignals sg_;
264
- std::unordered_map<void *, RankData *> buffers_;
265
- Signal *self_sg_;
261
+ std::unordered_map<void*, RankData*> buffers_;
262
+ Signal* self_sg_;
263
264
// stores the registered device pointers from all ranks
265
RankData *d_rank_data_base_, *d_rank_data_end_;
269
- std::vector<void *> graph_unreg_buffers_;
266
+ std::vector<void*> graph_unreg_buffers_;
267
// a map from IPC handles to opened IPC pointers
271
- std::map<IPC_KEY, char *> ipc_handles_;
268
+ std::map<IPC_KEY, char*> ipc_handles_;
269
270
/**
271
* meta is a pointer to device metadata and temporary buffer for allreduce.
@@ -279,22 +276,22 @@ class CustomAllreduce {
276
* note: this class does not own any device memory. Any required buffers
277
* are passed in from the constructor
278
*/
282
- CustomAllreduce(Signal *meta, void *rank_data, size_t rank_data_sz,
283
- const cudaIpcMemHandle_t *handles,
284
- const std::vector<int64_t> &offsets, int rank,
279
+ CustomAllreduce(Signal* meta, void* rank_data, size_t rank_data_sz,
280
+ const cudaIpcMemHandle_t* handles,
281
+ const std::vector<int64_t>& offsets, int rank,
282
bool full_nvlink = true)
283
: rank_(rank),
284
world_size_(offsets.size()),
285
full_nvlink_(full_nvlink),
286
self_sg_(meta),
290
- d_rank_data_base_(reinterpret_cast<RankData *>(rank_data)),
287
+ d_rank_data_base_(reinterpret_cast<RankData*>(rank_data)),
288
d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) {
289
for (int i = 0; i < world_size_; i++) {
293
- Signal *rank_sg;
290
+ Signal* rank_sg;
291
if (i != rank_) {
295
- char *handle = open_ipc_handle(&handles[i]);
292
+ char* handle = open_ipc_handle(&handles[i]);
293
handle += offsets[i];
297
- rank_sg = (Signal *)handle;
294
+ rank_sg = (Signal*)handle;
295
} else {
296
rank_sg = self_sg_;
297
}
@@ -302,13 +299,13 @@ class CustomAllreduce {
299
}
300
}
301
305
- char *open_ipc_handle(const void *ipc_handle) {
302
+ char* open_ipc_handle(const void* ipc_handle) {
303
auto [it, new_handle] =
307
- ipc_handles_.insert({*((IPC_KEY *)ipc_handle), nullptr});
304
+ ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr});
305
if (new_handle) {
309
- char *ipc_ptr;
310
- CUDACHECK(cudaIpcOpenMemHandle((void **)&ipc_ptr,
311
- *((const cudaIpcMemHandle_t *)ipc_handle),
306
+ char* ipc_ptr;
307
+ CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptr,
308
+ *((const cudaIpcMemHandle_t*)ipc_handle),
309
cudaIpcMemLazyEnablePeerAccess));
310
it->second = ipc_ptr;
311
}
@@ -323,7 +320,7 @@ class CustomAllreduce {
320
std::vector<int64_t> offsets(num_buffers);
321
for (int i = 0; i < num_buffers; i++) {
322
auto ptr = graph_unreg_buffers_[i];
326
- void *base_ptr;
323
+ void* base_ptr;
324
// note: must share the base address of each allocation, or we get wrong
325
// address
326
if (cuPointerGetAttribute(&base_ptr,
@@ -331,8 +328,8 @@ class CustomAllreduce {
328
(CUdeviceptr)ptr) != CUDA_SUCCESS)
329
throw std::runtime_error("failed to get pointer attr");
330
CUDACHECK(cudaIpcGetMemHandle(
334
- (cudaIpcMemHandle_t *)&handles[i * handle_sz], base_ptr));
335
- offsets[i] = ((char *)ptr) - ((char *)base_ptr);
331
+ (cudaIpcMemHandle_t*)&handles[i * handle_sz], base_ptr));
332
+ offsets[i] = ((char*)ptr) - ((char*)base_ptr);
333
}
334
return std::make_pair(handles, offsets);
335
}
@@ -344,13 +341,13 @@ class CustomAllreduce {
341
std::to_string(d_rank_data_base_ + num - d_rank_data_end_));
342
}
343
347
- void register_buffer(const std::vector<std::string> &handles,
348
- const std::vector<int64_t> &offsets, void *self) {
344
+ void register_buffer(const std::vector<std::string>& handles,
345
+ const std::vector<int64_t>& offsets, void* self) {
346
check_rank_data_capacity();
347
RankData data;
348
for (int i = 0; i < world_size_; i++) {
349
if (i != rank_) {
353
- char *handle = open_ipc_handle(handles[i].data());
350
+ char* handle = open_ipc_handle(handles[i].data());
351
handle += offsets[i];
352
data.ptrs[i] = handle;
353
} else {
@@ -371,17 +368,17 @@ class CustomAllreduce {
368
// got a different address. IPC handles have internal reference counting
369
// mechanism so overhead should be small.
370
void register_graph_buffers(
374
- const std::vector<std::string> &handles,
375
- const std::vector<std::vector<int64_t>> &offsets) {
371
+ const std::vector<std::string>& handles,
372
+ const std::vector<std::vector<int64_t>>& offsets) {
373
auto num_buffers = graph_unreg_buffers_.size();
374
check_rank_data_capacity(num_buffers);
375
std::vector<RankData> rank_data(num_buffers);
376
for (int i = 0; i < num_buffers; i++) {
377
auto self_ptr = graph_unreg_buffers_[i];
381
- auto &rd = rank_data[i];
378
+ auto& rd = rank_data[i];
379
for (int j = 0; j < world_size_; j++) {
380
if (j != rank_) {
384
- char *handle =
381
+ char* handle =
382
open_ipc_handle(&handles[j][i * sizeof(cudaIpcMemHandle_t)]);
383
handle += offsets[j][i];
384
rd.ptrs[j] = handle;
@@ -405,7 +402,7 @@ class CustomAllreduce {
402
* will cause contention on NVLink bus.
403
*/
404
template <typename T>
408
- void allreduce(cudaStream_t stream, T *input, T *output, int size,
405
+ void allreduce(cudaStream_t stream, T* input, T* output, int size,
406
int threads = 512, int block_limit = 36) {
407
auto d = packed_t<T>::P::size;
408
if (size % d != 0)
@@ -418,7 +415,7 @@ class CustomAllreduce {
415
std::to_string(kMaxBlocks) + ". Got " +
416
std::to_string(block_limit));
417
421
- RankData *ptrs;
418
+ RankData* ptrs;
419
cudaStreamCaptureStatus status;
420
CUDACHECK(cudaStreamIsCapturing(stream, &status));
421
if (status == cudaStreamCaptureStatusActive) {
csrc/custom_all_reduce_test.cu
+19
-19
@@ -48,7 +48,7 @@ __global__ void dummy_kernel() {
48
}
49
50
template <typename T>
51
-__global__ void set_data(T *data, int size, int myRank) {
51
+__global__ void set_data(T* data, int size, int myRank) {
52
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
53
idx += gridDim.x * blockDim.x) {
54
data[idx] = myRank * 0.11f;
@@ -56,8 +56,8 @@ __global__ void set_data(T *data, int size, int myRank) {
56
}
57
58
template <typename T>
59
-__global__ void convert_data(const T *data1, const T *data2, double *fdata1,
60
- double *fdata2, int size) {
59
+__global__ void convert_data(const T* data1, const T* data2, double* fdata1,
60
+ double* fdata2, int size) {
61
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
62
idx += gridDim.x * blockDim.x) {
63
fdata1[idx] = data1[idx];
@@ -65,7 +65,7 @@ __global__ void convert_data(const T *data1, const T *data2, double *fdata1,
65
}
66
}
67
68
-__global__ void init_rand(curandState_t *state, int size, int nRanks) {
68
+__global__ void init_rand(curandState_t* state, int size, int nRanks) {
69
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
70
idx += gridDim.x * blockDim.x) {
71
for (int i = 0; i < nRanks; i++) {
@@ -75,7 +75,7 @@ __global__ void init_rand(curandState_t *state, int size, int nRanks) {
75
}
76
77
template <typename T>
78
-__global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
78
+__global__ void gen_data(curandState_t* state, T* data, double* ground_truth,
79
int myRank, int nRanks, int size) {
80
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
81
idx += gridDim.x * blockDim.x) {
@@ -91,9 +91,9 @@ __global__ void gen_data(curandState_t *state, T *data, double *ground_truth,
91
}
92
93
template <typename T>
94
-void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
94
+void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
95
int data_size, bool performance_test) {
96
- T *result;
96
+ T* result;
97
cudaStream_t stream;
98
CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
99
CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
@@ -101,8 +101,8 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
101
102
cudaIpcMemHandle_t self_data_handle;
103
cudaIpcMemHandle_t data_handles[8];
104
- vllm::Signal *buffer;
105
- T *self_data_copy;
104
+ vllm::Signal* buffer;
105
+ T* self_data_copy;
106
/**
107
* Allocate IPC buffer
108
*
@@ -125,22 +125,22 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
125
MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
126
MPI_BYTE, MPI_COMM_WORLD));
127
128
- void *rank_data;
128
+ void* rank_data;
129
size_t rank_data_sz = 16 * 1024 * 1024;
130
CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
131
std::vector<int64_t> offsets(nRanks, 0);
132
vllm::CustomAllreduce fa(buffer, rank_data, rank_data_sz, data_handles,
133
offsets, myRank);
134
- auto *self_data =
135
- reinterpret_cast<T *>(reinterpret_cast<char *>(buffer) +
136
- sizeof(vllm::Signal) + data_size * sizeof(T));
134
+ auto* self_data =
135
+ reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
136
+ sizeof(vllm::Signal) + data_size * sizeof(T));
137
// hack buffer registration
138
{
139
std::vector<std::string> handles;
140
handles.reserve(nRanks);
141
for (int i = 0; i < nRanks; i++) {
142
- char *begin = (char *)&data_handles[i];
143
- char *end = (char *)&data_handles[i + 1];
142
+ char* begin = (char*)&data_handles[i];
143
+ char* end = (char*)&data_handles[i + 1];
144
handles.emplace_back(begin, end);
145
}
146
std::vector<int64_t> offsets(nRanks,
@@ -148,9 +148,9 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
148
fa.register_buffer(handles, offsets, self_data);
149
}
150
151
- double *ground_truth;
151
+ double* ground_truth;
152
CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
153
- curandState_t *states;
153
+ curandState_t* states;
154
CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
155
init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
156
gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
@@ -287,7 +287,7 @@ void run(int myRank, int nRanks, ncclComm_t &comm, int threads, int block_limit,
287
CUDACHECK(cudaStreamDestroy(stream));
288
}
289
290
-int main(int argc, char **argv) {
290
+int main(int argc, char** argv) {
291
int nRanks, myRank;
292
MPICHECK(MPI_Init(&argc, &argv));
293
MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
@@ -296,7 +296,7 @@ int main(int argc, char **argv) {
296
ncclUniqueId id;
297
ncclComm_t comm;
298
if (myRank == 0) ncclGetUniqueId(&id);
299
- MPICHECK(MPI_Bcast(static_cast<void *>(&id), sizeof(id), MPI_BYTE, 0,
299
+ MPICHECK(MPI_Bcast(static_cast<void*>(&id), sizeof(id), MPI_BYTE, 0,
300
MPI_COMM_WORLD));
301
NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
302
csrc/dispatch_utils.h
+20
-22
@@ -6,32 +6,30 @@
6
7
#include <torch/extension.h>
8
9
-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
11
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
9
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
11
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
12
AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
13
14
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15
- AT_DISPATCH_SWITCH( \
16
- TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
14
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
16
18
-#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
19
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
20
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
21
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
17
+#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
18
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
19
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
20
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
21
AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
22
24
-#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25
- AT_DISPATCH_SWITCH( \
26
- TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
27
-
28
-#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
29
- AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
30
- AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
31
- AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
32
- AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
23
+#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
24
+ AT_DISPATCH_SWITCH(TYPE, NAME, \
25
+ VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
26
+
27
+#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
28
+ AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
29
+ AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
30
+ AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
31
+ AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
32
AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__)
33
35
-#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
36
- AT_DISPATCH_SWITCH( \
37
- TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
34
+#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
35
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
csrc/layernorm_kernels.cu
+121
-121
@@ -11,26 +11,24 @@
11
#include <hip/hip_bf16.h>
12
#include <hip/hip_fp16.h>
13
14
- using __nv_bfloat16 = __hip_bfloat16;
15
- using __nv_bfloat162 = __hip_bfloat162;
14
+using __nv_bfloat16 = __hip_bfloat16;
15
+using __nv_bfloat162 = __hip_bfloat162;
16
#endif
17
18
namespace vllm {
19
20
// TODO(woosuk): Further optimize this kernel.
21
-template<typename scalar_t>
21
+template <typename scalar_t>
22
__global__ void rms_norm_kernel(
23
- scalar_t* __restrict__ out, // [..., hidden_size]
24
- const scalar_t* __restrict__ input, // [..., hidden_size]
25
- const scalar_t* __restrict__ weight, // [hidden_size]
26
- const float epsilon,
27
- const int num_tokens,
28
- const int hidden_size) {
23
+ scalar_t* __restrict__ out, // [..., hidden_size]
24
+ const scalar_t* __restrict__ input, // [..., hidden_size]
25
+ const scalar_t* __restrict__ weight, // [hidden_size]
26
+ const float epsilon, const int num_tokens, const int hidden_size) {
27
__shared__ float s_variance;
28
float variance = 0.0f;
29
30
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
33
- const float x = (float) input[blockIdx.x * hidden_size + idx];
31
+ const float x = (float)input[blockIdx.x * hidden_size + idx];
32
variance += x * x;
33
}
34
variance = blockReduceSum<float>(variance);
@@ -40,12 +38,12 @@ __global__ void rms_norm_kernel(
38
__syncthreads();
39
40
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
43
- float x = (float) input[blockIdx.x * hidden_size + idx];
44
- out[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
41
+ float x = (float)input[blockIdx.x * hidden_size + idx];
42
+ out[blockIdx.x * hidden_size + idx] =
43
+ ((scalar_t)(x * s_variance)) * weight[idx];
44
}
45
}
46
48
-
47
/* Converter structs for the conversion from torch types to HIP/CUDA types,
48
and the associated type conversions within HIP/CUDA. These helpers need
49
to be implemented for now because the relevant type conversion
@@ -54,51 +52,68 @@ __global__ void rms_norm_kernel(
52
53
Each struct should have the member static constexpr bool `exists`:
54
If false, the optimized kernel is not used for the corresponding torch type.
57
- If true, the struct should be fully defined as shown in the examples below.
55
+ If true, the struct should be fully defined as shown in the examples below.
56
*/
59
-template<typename torch_type>
60
-struct _typeConvert { static constexpr bool exists = false; };
57
+template <typename torch_type>
58
+struct _typeConvert {
59
+ static constexpr bool exists = false;
60
+};
61
62
#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
63
// CUDA < 12.0 runs into issues with packed type conversion
64
-template<>
64
+template <>
65
struct _typeConvert<c10::Half> {
66
static constexpr bool exists = true;
67
using hip_type = __half;
68
using packed_hip_type = __half2;
69
70
__device__ static inline float convert(hip_type x) { return __half2float(x); }
71
- __device__ static inline float2 convert(packed_hip_type x) { return __half22float2(x); }
72
- __device__ static inline hip_type convert(float x) { return __float2half_rn(x); }
73
- __device__ static inline packed_hip_type convert(float2 x) { return __float22half2_rn(x); }
71
+ __device__ static inline float2 convert(packed_hip_type x) {
72
+ return __half22float2(x);
73
+ }
74
+ __device__ static inline hip_type convert(float x) {
75
+ return __float2half_rn(x);
76
+ }
77
+ __device__ static inline packed_hip_type convert(float2 x) {
78
+ return __float22half2_rn(x);
79
+ }
80
};
81
76
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
82
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
83
// CUDA_ARCH < 800 does not have BF16 support
84
// TODO: Add in ROCm support once public headers handle bf16 maturely
79
-template<>
85
+template <>
86
struct _typeConvert<c10::BFloat16> {
87
static constexpr bool exists = true;
88
using hip_type = __nv_bfloat16;
89
using packed_hip_type = __nv_bfloat162;
90
85
- __device__ static inline float convert(hip_type x) { return __bfloat162float(x); }
86
- __device__ static inline float2 convert(packed_hip_type x) { return __bfloat1622float2(x); }
87
- __device__ static inline hip_type convert(float x) { return __float2bfloat16(x); }
88
- __device__ static inline packed_hip_type convert(float2 x) { return __float22bfloat162_rn(x); }
91
+ __device__ static inline float convert(hip_type x) {
92
+ return __bfloat162float(x);
93
+ }
94
+ __device__ static inline float2 convert(packed_hip_type x) {
95
+ return __bfloat1622float2(x);
96
+ }
97
+ __device__ static inline hip_type convert(float x) {
98
+ return __float2bfloat16(x);
99
+ }
100
+ __device__ static inline packed_hip_type convert(float2 x) {
101
+ return __float22bfloat162_rn(x);
102
+ }
103
};
90
-#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
91
-#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
104
+ #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
105
+#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
106
+ // 12000))
107
108
/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
109
for appropriate specializations of fused_add_rms_norm_kernel.
110
Only functions that are necessary in that kernel are implemented.
111
Alignment to 16 bytes is required to use 128-bit global memory ops.
112
*/
98
-template<typename scalar_t, int width>
113
+template <typename scalar_t, int width>
114
struct alignas(16) _f16Vec {
100
- /* Not theoretically necessary that width is a power of 2 but should
101
- almost always be the case for optimization purposes */
115
+ /* Not theoretically necessary that width is a power of 2 but should
116
+ almost always be the case for optimization purposes */
117
static_assert(width > 0 && (width & (width - 1)) == 0,
118
"Width is not a positive power of 2!");
119
using Converter = _typeConvert<scalar_t>;
@@ -108,51 +123,49 @@ struct alignas(16) _f16Vec {
123
124
__device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
125
if constexpr (width % 2 == 0) {
111
- #pragma unroll
126
+#pragma unroll
127
for (int i = 0; i < width; i += 2) {
113
- T2 temp{data[i], data[i+1]};
114
- temp += T2{other.data[i], other.data[i+1]};
128
+ T2 temp{data[i], data[i + 1]};
129
+ temp += T2{other.data[i], other.data[i + 1]};
130
data[i] = temp.x;
116
- data[i+1] = temp.y;
131
+ data[i + 1] = temp.y;
132
}
133
} else {
119
- #pragma unroll
120
- for (int i = 0; i < width; ++i)
121
- data[i] += other.data[i];
134
+#pragma unroll
135
+ for (int i = 0; i < width; ++i) data[i] += other.data[i];
136
}
137
return *this;
138
}
139
140
__device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
141
if constexpr (width % 2 == 0) {
128
- #pragma unroll
142
+#pragma unroll
143
for (int i = 0; i < width; i += 2) {
130
- T2 temp{data[i], data[i+1]};
131
- temp *= T2{other.data[i], other.data[i+1]};
144
+ T2 temp{data[i], data[i + 1]};
145
+ temp *= T2{other.data[i], other.data[i + 1]};
146
data[i] = temp.x;
133
- data[i+1] = temp.y;
147
+ data[i + 1] = temp.y;
148
}
149
} else {
136
- #pragma unroll
137
- for (int i = 0; i < width; ++i)
138
- data[i] *= other.data[i];
150
+#pragma unroll
151
+ for (int i = 0; i < width; ++i) data[i] *= other.data[i];
152
}
153
return *this;
154
}
155
156
__device__ _f16Vec& operator*=(const float scale) {
157
if constexpr (width % 2 == 0) {
145
- #pragma unroll
158
+#pragma unroll
159
for (int i = 0; i < width; i += 2) {
147
- float2 temp_f = Converter::convert(T2{data[i], data[i+1]});
160
+ float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
161
temp_f.x *= scale;
162
temp_f.y *= scale;
163
T2 temp = Converter::convert(temp_f);
164
data[i] = temp.x;
152
- data[i+1] = temp.y;
165
+ data[i + 1] = temp.y;
166
}
167
} else {
155
- #pragma unroll
168
+#pragma unroll
169
for (int i = 0; i < width; ++i) {
170
float temp = Converter::convert(data[i]) * scale;
171
data[i] = Converter::convert(temp);
@@ -164,13 +177,13 @@ struct alignas(16) _f16Vec {
177
__device__ float sum_squares() const {
178
float result = 0.0f;
179
if constexpr (width % 2 == 0) {
167
- #pragma unroll
180
+#pragma unroll
181
for (int i = 0; i < width; i += 2) {
169
- float2 z = Converter::convert(T2{data[i], data[i+1]});
182
+ float2 z = Converter::convert(T2{data[i], data[i + 1]});
183
result += z.x * z.x + z.y * z.y;
184
}
185
} else {
173
- #pragma unroll
186
+#pragma unroll
187
for (int i = 0; i < width; ++i) {
188
float x = Converter::convert(data[i]);
189
result += x * x;
@@ -184,15 +197,13 @@ struct alignas(16) _f16Vec {
197
Additional optimizations we can make in this case are
198
packed and vectorized operations, which help with the
199
memory latency bottleneck. */
187
-template<typename scalar_t, int width>
188
-__global__ std::enable_if_t<
189
- (width > 0) && _typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
190
- scalar_t* __restrict__ input, // [..., hidden_size]
191
- scalar_t* __restrict__ residual, // [..., hidden_size]
192
- const scalar_t* __restrict__ weight, // [hidden_size]
193
- const float epsilon,
194
- const int num_tokens,
195
- const int hidden_size) {
200
+template <typename scalar_t, int width>
201
+__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
202
+fused_add_rms_norm_kernel(
203
+ scalar_t* __restrict__ input, // [..., hidden_size]
204
+ scalar_t* __restrict__ residual, // [..., hidden_size]
205
+ const scalar_t* __restrict__ weight, // [hidden_size]
206
+ const float epsilon, const int num_tokens, const int hidden_size) {
207
// Sanity checks on our vector struct and type-punned pointer arithmetic
208
static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
209
static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
@@ -203,9 +214,12 @@ __global__ std::enable_if_t<
214
/* These and the argument pointers are all declared `restrict` as they are
215
not aliased in practice. Argument pointers should not be dereferenced
216
in this kernel as that would be undefined behavior */
206
- auto* __restrict__ input_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
207
- auto* __restrict__ residual_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
208
- auto* __restrict__ weight_v = reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
217
+ auto* __restrict__ input_v =
218
+ reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
219
+ auto* __restrict__ residual_v =
220
+ reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
221
+ auto* __restrict__ weight_v =
222
+ reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
223
224
for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
225
int id = blockIdx.x * vec_hidden_size + idx;
@@ -215,10 +229,11 @@ __global__ std::enable_if_t<
229
residual_v[id] = temp;
230
}
231
/* Keep the following if-else block in sync with the
218
- calculation of max_block_size in fused_add_rms_norm */
232
+ calculation of max_block_size in fused_add_rms_norm */
233
if (num_tokens < 256) {
234
variance = blockReduceSum<float, 1024>(variance);
221
- } else variance = blockReduceSum<float, 256>(variance);
235
+ } else
236
+ variance = blockReduceSum<float, 256>(variance);
237
if (threadIdx.x == 0) {
238
s_variance = rsqrtf(variance / hidden_size + epsilon);
239
}
@@ -233,52 +248,50 @@ __global__ std::enable_if_t<
248
}
249
}
250
236
-
251
/* Generic fused_add_rms_norm_kernel
252
The width field is not used here but necessary for other specializations.
253
*/
240
-template<typename scalar_t, int width>
241
-__global__ std::enable_if_t<
242
- (width == 0) || !_typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
243
- scalar_t* __restrict__ input, // [..., hidden_size]
244
- scalar_t* __restrict__ residual, // [..., hidden_size]
245
- const scalar_t* __restrict__ weight, // [hidden_size]
246
- const float epsilon,
247
- const int num_tokens,
248
- const int hidden_size) {
254
+template <typename scalar_t, int width>
255
+__global__ std::enable_if_t<(width == 0) || !_typeConvert<scalar_t>::exists>
256
+fused_add_rms_norm_kernel(
257
+ scalar_t* __restrict__ input, // [..., hidden_size]
258
+ scalar_t* __restrict__ residual, // [..., hidden_size]
259
+ const scalar_t* __restrict__ weight, // [hidden_size]
260
+ const float epsilon, const int num_tokens, const int hidden_size) {
261
__shared__ float s_variance;
262
float variance = 0.0f;
263
264
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
265
scalar_t z = input[blockIdx.x * hidden_size + idx];
266
z += residual[blockIdx.x * hidden_size + idx];
255
- float x = (float) z;
267
+ float x = (float)z;
268
variance += x * x;
269
residual[blockIdx.x * hidden_size + idx] = z;
270
}
271
/* Keep the following if-else block in sync with the
260
- calculation of max_block_size in fused_add_rms_norm */
272
+ calculation of max_block_size in fused_add_rms_norm */
273
if (num_tokens < 256) {
274
variance = blockReduceSum<float, 1024>(variance);
263
- } else variance = blockReduceSum<float, 256>(variance);
275
+ } else
276
+ variance = blockReduceSum<float, 256>(variance);
277
if (threadIdx.x == 0) {
278
s_variance = rsqrtf(variance / hidden_size + epsilon);
279
}
280
__syncthreads();
281
282
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
270
- float x = (float) residual[blockIdx.x * hidden_size + idx];
271
- input[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
283
+ float x = (float)residual[blockIdx.x * hidden_size + idx];
284
+ input[blockIdx.x * hidden_size + idx] =
285
+ ((scalar_t)(x * s_variance)) * weight[idx];
286
}
287
}
288
275
-} // namespace vllm
289
+} // namespace vllm
290
277
-void rms_norm(
278
- torch::Tensor& out, // [..., hidden_size]
279
- torch::Tensor& input, // [..., hidden_size]
280
- torch::Tensor& weight, // [hidden_size]
281
- float epsilon) {
291
+void rms_norm(torch::Tensor& out, // [..., hidden_size]
292
+ torch::Tensor& input, // [..., hidden_size]
293
+ torch::Tensor& weight, // [hidden_size]
294
+ float epsilon) {
295
int hidden_size = input.size(-1);
296
int num_tokens = input.numel() / hidden_size;
297
@@ -286,40 +299,27 @@ void rms_norm(
299
dim3 block(std::min(hidden_size, 1024));
300
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
301
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
289
- VLLM_DISPATCH_FLOATING_TYPES(
290
- input.scalar_type(),
291
- "rms_norm_kernel",
292
- [&] {
293
- vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
294
- out.data_ptr<scalar_t>(),
295
- input.data_ptr<scalar_t>(),
296
- weight.data_ptr<scalar_t>(),
297
- epsilon,
298
- num_tokens,
299
- hidden_size);
300
- });
302
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_kernel", [&] {
303
+ vllm::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
304
+ out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
305
+ weight.data_ptr<scalar_t>(), epsilon, num_tokens, hidden_size);
306
+ });
307
}
308
303
-#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
304
- VLLM_DISPATCH_FLOATING_TYPES( \
305
- input.scalar_type(), \
306
- "fused_add_rms_norm_kernel", \
307
- [&] { \
308
- vllm::fused_add_rms_norm_kernel \
309
- <scalar_t, width><<<grid, block, 0, stream>>>( \
310
- input.data_ptr<scalar_t>(), \
311
- residual.data_ptr<scalar_t>(), \
312
- weight.data_ptr<scalar_t>(), \
313
- epsilon, \
314
- num_tokens, \
315
- hidden_size); \
316
- });
317
-
318
-void fused_add_rms_norm(
319
- torch::Tensor& input, // [..., hidden_size]
320
- torch::Tensor& residual, // [..., hidden_size]
321
- torch::Tensor& weight, // [hidden_size]
322
- float epsilon) {
309
+#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
310
+ VLLM_DISPATCH_FLOATING_TYPES( \
311
+ input.scalar_type(), "fused_add_rms_norm_kernel", [&] { \
312
+ vllm::fused_add_rms_norm_kernel<scalar_t, width> \
313
+ <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(), \
314
+ residual.data_ptr<scalar_t>(), \
315
+ weight.data_ptr<scalar_t>(), epsilon, \
316
+ num_tokens, hidden_size); \
317
+ });
318
+
319
+void fused_add_rms_norm(torch::Tensor& input, // [..., hidden_size]
320
+ torch::Tensor& residual, // [..., hidden_size]
321
+ torch::Tensor& weight, // [hidden_size]
322
+ float epsilon) {
323
int hidden_size = input.size(-1);
324
int num_tokens = input.numel() / hidden_size;
325
@@ -342,8 +342,8 @@ void fused_add_rms_norm(
342
auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
343
auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
344
auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
345
- bool ptrs_are_aligned = inp_ptr % 16 == 0 && res_ptr % 16 == 0 \
346
- && wt_ptr % 16 == 0;
345
+ bool ptrs_are_aligned =
346
+ inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
347
if (ptrs_are_aligned && hidden_size % 8 == 0) {
348
LAUNCH_FUSED_ADD_RMS_NORM(8);
349
} else {
csrc/moe/moe_ops.cpp
+2
-1
@@ -3,5 +3,6 @@
3
#include <torch/extension.h>
4
5
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6
- m.def("topk_softmax", &topk_softmax, "Apply topk softmax to the gating outputs.");
6
+ m.def("topk_softmax", &topk_softmax,
7
+ "Apply topk softmax to the gating outputs.");
8
}
csrc/moe/moe_ops.h
+3
-5
@@ -2,8 +2,6 @@
2
3
#include <torch/extension.h>
4
5
-void topk_softmax(
6
- torch::Tensor& topk_weights,
7
- torch::Tensor& topk_indices,
8
- torch::Tensor& token_expert_indices,
9
- torch::Tensor& gating_output);
5
+void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices,
6
+ torch::Tensor& token_expert_indices,
7
+ torch::Tensor& gating_output);
csrc/moe_align_block_size_kernels.cu
+110
-101
@@ -7,119 +7,128 @@
7
#include "cuda_compat.h"
8
#include "dispatch_utils.h"
9
10
-#define CEILDIV(x,y) (((x) + (y) - 1) / (y))
10
+#define CEILDIV(x, y) (((x) + (y) - 1) / (y))
11
12
namespace vllm {
13
14
namespace {
15
-__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row, int32_t col) {
16
- // don't worry about overflow because num_experts is relatively small
17
- return row * total_col + col;
18
-}
15
+__device__ __forceinline__ int32_t index(int32_t total_col, int32_t row,
16
+ int32_t col) {
17
+ // don't worry about overflow because num_experts is relatively small
18
+ return row * total_col + col;
19
}
20
+} // namespace
21
22
template <typename scalar_t>
22
-__global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
23
- int32_t *sorted_token_ids,
24
- int32_t *expert_ids,
25
- int32_t *total_tokens_post_pad,
26
- int32_t num_experts,
27
- int32_t block_size,
28
- size_t numel) {
29
- const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30
- const size_t start_idx = threadIdx.x * tokens_per_thread;
31
-
32
- extern __shared__ int32_t shared_mem[];
33
-
34
- int32_t* tokens_cnts = shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
35
- int32_t* cumsum = shared_mem + (num_experts + 1) * num_experts; // 1d tensor with shape (num_experts + 1)
36
-
37
- for (int i = 0; i < num_experts; ++i) {
38
- tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
39
- }
40
-
41
- /**
42
- * In the first step we compute token_cnts[thread_index + 1][expert_index],
43
- * which counts how many tokens in the token shard of thread_index are assigned
44
- * to expert expert_index.
45
- */
46
- for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
47
- ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
48
- }
49
-
50
- __syncthreads();
51
-
52
- // For each expert we accumulate the token counts from the different threads.
53
- tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
54
- for (int i = 1; i <= blockDim.x; ++i) {
55
- tokens_cnts[index(num_experts, i, threadIdx.x)] += tokens_cnts[index(num_experts, i-1, threadIdx.x)];
56
- }
57
-
58
- __syncthreads();
59
-
60
- // We accumulate the token counts of all experts in thread 0.
61
- if (threadIdx.x == 0) {
62
- cumsum[0] = 0;
63
- for (int i = 1; i <= num_experts; ++i) {
64
- cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)], block_size) * block_size;
65
- }
66
- *total_tokens_post_pad = cumsum[num_experts];
67
- }
68
-
69
- __syncthreads();
70
-
71
- /**
72
- * For each expert, each thread processes the tokens of the corresponding blocks
73
- * and stores the corresponding expert_id for each block.
74
- */
75
- for (int i = cumsum[threadIdx.x];i < cumsum[threadIdx.x + 1];i += block_size) {
76
- expert_ids[i / block_size] = threadIdx.x;
23
+__global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
24
+ int32_t* sorted_token_ids,
25
+ int32_t* expert_ids,
26
+ int32_t* total_tokens_post_pad,
27
+ int32_t num_experts,
28
+ int32_t block_size, size_t numel) {
29
+ const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
30
+ const size_t start_idx = threadIdx.x * tokens_per_thread;
31
+
32
+ extern __shared__ int32_t shared_mem[];
33
+
34
+ int32_t* tokens_cnts =
35
+ shared_mem; // 2d tensor with shape (num_experts + 1, num_experts)
36
+ int32_t* cumsum =
37
+ shared_mem + (num_experts + 1) *
38
+ num_experts; // 1d tensor with shape (num_experts + 1)
39
+
40
+ for (int i = 0; i < num_experts; ++i) {
41
+ tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
42
+ }
43
+
44
+ /**
45
+ * In the first step we compute token_cnts[thread_index + 1][expert_index],
46
+ * which counts how many tokens in the token shard of thread_index are
47
+ * assigned to expert expert_index.
48
+ */
49
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
50
+ ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
51
+ }
52
+
53
+ __syncthreads();
54
+
55
+ // For each expert we accumulate the token counts from the different threads.
56
+ tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
57
+ for (int i = 1; i <= blockDim.x; ++i) {
58
+ tokens_cnts[index(num_experts, i, threadIdx.x)] +=
59
+ tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
60
+ }
61
+
62
+ __syncthreads();
63
+
64
+ // We accumulate the token counts of all experts in thread 0.
65
+ if (threadIdx.x == 0) {
66
+ cumsum[0] = 0;
67
+ for (int i = 1; i <= num_experts; ++i) {
68
+ cumsum[i] = cumsum[i - 1] +
69
+ CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)],
70
+ block_size) *
71
+ block_size;
72
}
78
-
79
- /**
80
- * Each thread processes a token shard, calculating the index of each token after
81
- * sorting by expert number. Given the example topk_ids = [0,1,2,1,2,3,0,3,4] and
82
- * block_size = 4, then the output would be [0, 6, *, *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *],
83
- * where * represents a padding value(preset in python).
84
- */
85
- for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
86
- int32_t expert_id = topk_ids[i];
87
- /** The cumsum[expert_id] stores the starting index of the tokens that the
88
- * expert with expert_id needs to process, and tokens_cnts[threadIdx.x][expert_id]
89
- * stores the indices of the tokens processed by the expert with expert_id within
90
- * the current thread's token shard.
91
- */
92
- int32_t rank_post_pad = tokens_cnts[index(num_experts, threadIdx.x, expert_id)] + cumsum[expert_id];
93
- sorted_token_ids[rank_post_pad] = i;
94
- ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
95
- }
96
-}
73
+ *total_tokens_post_pad = cumsum[num_experts];
74
+ }
75
+
76
+ __syncthreads();
77
+
78
+ /**
79
+ * For each expert, each thread processes the tokens of the corresponding
80
+ * blocks and stores the corresponding expert_id for each block.
81
+ */
82
+ for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
83
+ i += block_size) {
84
+ expert_ids[i / block_size] = threadIdx.x;
85
+ }
86
+
87
+ /**
88
+ * Each thread processes a token shard, calculating the index of each token
89
+ * after sorting by expert number. Given the example topk_ids =
90
+ * [0,1,2,1,2,3,0,3,4] and block_size = 4, then the output would be [0, 6, *,
91
+ * *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *], where * represents a
92
+ * padding value(preset in python).
93
+ */
94
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
95
+ int32_t expert_id = topk_ids[i];
96
+ /** The cumsum[expert_id] stores the starting index of the tokens that the
97
+ * expert with expert_id needs to process, and
98
+ * tokens_cnts[threadIdx.x][expert_id] stores the indices of the tokens
99
+ * processed by the expert with expert_id within the current thread's token
100
+ * shard.
101
+ */
102
+ int32_t rank_post_pad =
103
+ tokens_cnts[index(num_experts, threadIdx.x, expert_id)] +
104
+ cumsum[expert_id];
105
+ sorted_token_ids[rank_post_pad] = i;
106
+ ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
107
+ }
108
}
98
-
99
-void moe_align_block_size(
100
- torch::Tensor topk_ids,
101
- int num_experts,
102
- int block_size,
103
- torch::Tensor sorted_token_ids,
104
- torch::Tensor experts_ids,
105
- torch::Tensor num_tokens_post_pad) {
106
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
107
- VLLM_DISPATCH_INTEGRAL_TYPES(
108
- topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
109
- // calc needed amount of shared mem for `tokens_cnts` and `cumsum` tensors
110
- const int32_t shared_mem = ((num_experts + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
109
+} // namespace vllm
110
+
111
+void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
112
+ int block_size, torch::Tensor sorted_token_ids,
113
+ torch::Tensor experts_ids,
114
+ torch::Tensor num_tokens_post_pad) {
115
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
116
+ VLLM_DISPATCH_INTEGRAL_TYPES(
117
+ topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
118
+ // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
119
+ // tensors
120
+ const int32_t shared_mem =
121
+ ((num_experts + 1) * num_experts + (num_experts + 1)) *
122
+ sizeof(int32_t);
123
124
// set dynamic shared mem
125
auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114
- AT_CUDA_CHECK(
115
- VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize((void *)kernel, shared_mem));
126
+ AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
127
+ (void*)kernel, shared_mem));
128
kernel<<<1, num_experts, shared_mem, stream>>>(
117
- topk_ids.data_ptr<scalar_t>(),
118
- sorted_token_ids.data_ptr<int32_t>(),
119
- experts_ids.data_ptr<int32_t>(),
120
- num_tokens_post_pad.data_ptr<int32_t>(),
121
- num_experts,
122
- block_size,
129
+ topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
130
+ experts_ids.data_ptr<int32_t>(),
131
+ num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
132
topk_ids.numel());
124
- });
133
+ });
134
}
csrc/ops.h
+121
-209
@@ -2,224 +2,136 @@
2
3
#include <torch/extension.h>
4
5
-void paged_attention_v1(
6
- torch::Tensor& out,
7
- torch::Tensor& query,
8
- torch::Tensor& key_cache,
9
- torch::Tensor& value_cache,
10
- int num_kv_heads,
11
- float scale,
12
- torch::Tensor& block_tables,
13
- torch::Tensor& seq_lens,
14
- int block_size,
15
- int max_seq_len,
16
- const c10::optional<torch::Tensor>& alibi_slopes,
17
- const std::string& kv_cache_dtype,
18
- float kv_scale);
19
-
20
-void paged_attention_v2(
21
- torch::Tensor& out,
22
- torch::Tensor& exp_sums,
23
- torch::Tensor& max_logits,
24
- torch::Tensor& tmp_out,
25
- torch::Tensor& query,
26
- torch::Tensor& key_cache,
27
- torch::Tensor& value_cache,
28
- int num_kv_heads,
29
- float scale,
30
- torch::Tensor& block_tables,
31
- torch::Tensor& seq_lens,
32
- int block_size,
33
- int max_seq_len,
34
- const c10::optional<torch::Tensor>& alibi_slopes,
35
- const std::string& kv_cache_dtype,
36
- float kv_scale);
37
-
38
-void rms_norm(
39
- torch::Tensor& out,
40
- torch::Tensor& input,
41
- torch::Tensor& weight,
42
- float epsilon);
43
-
44
-void fused_add_rms_norm(
45
- torch::Tensor& input,
46
- torch::Tensor& residual,
47
- torch::Tensor& weight,
48
- float epsilon);
49
-
50
-void rotary_embedding(
51
- torch::Tensor& positions,
52
- torch::Tensor& query,
53
- torch::Tensor& key,
54
- int head_size,
55
- torch::Tensor& cos_sin_cache,
56
- bool is_neox);
57
-
58
-void batched_rotary_embedding(
59
- torch::Tensor& positions,
60
- torch::Tensor& query,
61
- torch::Tensor& key,
62
- int head_size,
63
- torch::Tensor& cos_sin_cache,
64
- bool is_neox,
65
- int rot_dim,
66
- torch::Tensor& cos_sin_cache_offsets);
67
-
68
-void silu_and_mul(
69
- torch::Tensor& out,
70
- torch::Tensor& input);
71
-
72
-void gelu_and_mul(
73
- torch::Tensor& out,
74
- torch::Tensor& input);
75
-
76
-void gelu_tanh_and_mul(
77
- torch::Tensor& out,
78
- torch::Tensor& input);
79
-
80
-void gelu_new(
81
- torch::Tensor& out,
82
- torch::Tensor& input);
83
-
84
-void gelu_fast(
85
- torch::Tensor& out,
86
- torch::Tensor& input);
5
+void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
6
+ torch::Tensor& key_cache, torch::Tensor& value_cache,
7
+ int num_kv_heads, float scale,
8
+ torch::Tensor& block_tables, torch::Tensor& seq_lens,
9
+ int block_size, int max_seq_len,
10
+ const c10::optional<torch::Tensor>& alibi_slopes,
11
+ const std::string& kv_cache_dtype, float kv_scale);
12
+
13
+void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
14
+ torch::Tensor& max_logits, torch::Tensor& tmp_out,
15
+ torch::Tensor& query, torch::Tensor& key_cache,
16
+ torch::Tensor& value_cache, int num_kv_heads,
17
+ float scale, torch::Tensor& block_tables,
18
+ torch::Tensor& seq_lens, int block_size,
19
+ int max_seq_len,
20
+ const c10::optional<torch::Tensor>& alibi_slopes,
21
+ const std::string& kv_cache_dtype, float kv_scale);
22
+
23
+void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
24
+ float epsilon);
25
+
26
+void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
27
+ torch::Tensor& weight, float epsilon);
28
+
29
+void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
30
+ torch::Tensor& key, int head_size,
31
+ torch::Tensor& cos_sin_cache, bool is_neox);
32
+
33
+void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
34
+ torch::Tensor& key, int head_size,
35
+ torch::Tensor& cos_sin_cache, bool is_neox,
36
+ int rot_dim,
37
+ torch::Tensor& cos_sin_cache_offsets);
38
+
39
+void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
40
+
41
+void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
42
+
43
+void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
44
+
45
+void gelu_new(torch::Tensor& out, torch::Tensor& input);
46
+
47
+void gelu_fast(torch::Tensor& out, torch::Tensor& input);
48
49
#ifndef USE_ROCM
89
-torch::Tensor aqlm_gemm(
90
- const torch::Tensor& input,
91
- const torch::Tensor& codes,
92
- const torch::Tensor& codebooks,
93
- const torch::Tensor& scales,
94
- const torch::Tensor& codebook_partition_sizes,
95
- const std::optional<torch::Tensor>& bias
96
-);
97
-
98
-torch::Tensor aqlm_dequant(
99
- const torch::Tensor& codes,
100
- const torch::Tensor& codebooks,
101
- const torch::Tensor& codebook_partition_sizes
102
-);
103
-
104
-torch::Tensor awq_gemm(
105
- torch::Tensor _in_feats,
106
- torch::Tensor _kernel,
107
- torch::Tensor _scaling_factors,
108
- torch::Tensor _zeros,
109
- int split_k_iters);
110
-
111
-torch::Tensor awq_dequantize(
112
- torch::Tensor _kernel,
113
- torch::Tensor _scaling_factors,
114
- torch::Tensor _zeros,
115
- int split_k_iters,
116
- int thx,
117
- int thy);
118
-
119
-torch::Tensor marlin_gemm(
120
- torch::Tensor& a,
121
- torch::Tensor& b_q_weight,
122
- torch::Tensor& b_scales,
123
- torch::Tensor& workspace,
124
- int64_t size_m,
125
- int64_t size_n,
126
- int64_t size_k);
127
-
128
-torch::Tensor gptq_marlin_24_gemm(
129
- torch::Tensor &a,
130
- torch::Tensor &b_q_weight,
131
- torch::Tensor &b_meta,
132
- torch::Tensor &b_scales,
133
- torch::Tensor &workspace,
134
- int64_t num_bits,
135
- int64_t size_m,
136
- int64_t size_n,
137
- int64_t size_k);
138
-
139
-torch::Tensor gptq_marlin_gemm(
140
- torch::Tensor &a,
141
- torch::Tensor &b_q_weight,
142
- torch::Tensor &b_scales,
143
- torch::Tensor &g_idx,
144
- torch::Tensor &perm,
145
- torch::Tensor &workspace,
146
- int64_t num_bits,
147
- int64_t size_m,
148
- int64_t size_n,
149
- int64_t size_k,
150
- bool is_k_full);
151
-
152
-torch::Tensor gptq_marlin_repack(
153
- torch::Tensor &b_q_weight,
154
- torch::Tensor &perm,
155
- int64_t size_k,
156
- int64_t size_n,
157
- int64_t num_bits);
158
-
159
-int cutlass_scaled_mm_dq(
160
- torch::Tensor& out,
161
- torch::Tensor const &a,
162
- torch::Tensor const &b,
163
- torch::Tensor const &a_scales,
164
- torch::Tensor const &b_scales);
50
+torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
51
+ const torch::Tensor& codebooks,
52
+ const torch::Tensor& scales,
53
+ const torch::Tensor& codebook_partition_sizes,
54
+ const std::optional<torch::Tensor>& bias);
55
+
56
+torch::Tensor aqlm_dequant(const torch::Tensor& codes,
57
+ const torch::Tensor& codebooks,
58
+ const torch::Tensor& codebook_partition_sizes);
59
+
60
+torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
61
+ torch::Tensor _scaling_factors, torch::Tensor _zeros,
62
+ int split_k_iters);
63
+
64
+torch::Tensor awq_dequantize(torch::Tensor _kernel,
65
+ torch::Tensor _scaling_factors,
66
+ torch::Tensor _zeros, int split_k_iters, int thx,
67
+ int thy);
68
+
69
+torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
70
+ torch::Tensor& b_scales, torch::Tensor& workspace,
71
+ int64_t size_m, int64_t size_n, int64_t size_k);
72
+
73
+torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
74
+ torch::Tensor& b_meta,
75
+ torch::Tensor& b_scales,
76
+ torch::Tensor& workspace, int64_t num_bits,
77
+ int64_t size_m, int64_t size_n,
78
+ int64_t size_k);
79
+
80
+torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
81
+ torch::Tensor& b_scales, torch::Tensor& g_idx,
82
+ torch::Tensor& perm, torch::Tensor& workspace,
83
+ int64_t num_bits, int64_t size_m, int64_t size_n,
84
+ int64_t size_k, bool is_k_full);
85
+
86
+torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
87
+ int64_t size_k, int64_t size_n,
88
+ int64_t num_bits);
89
+
90
+int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
91
+ torch::Tensor const& b, torch::Tensor const& a_scales,
92
+ torch::Tensor const& b_scales);
93
94
#endif
95
168
-void squeezellm_gemm(
169
- torch::Tensor vec,
170
- torch::Tensor mat,
171
- torch::Tensor mul,
172
- torch::Tensor lookup_table);
173
-
174
-torch::Tensor gptq_gemm(
175
- torch::Tensor a,
176
- torch::Tensor b_q_weight,
177
- torch::Tensor b_gptq_qzeros,
178
- torch::Tensor b_gptq_scales,
179
- torch::Tensor b_g_idx,
180
- bool use_exllama,
181
- int bit);
182
-
183
-void gptq_shuffle(
184
- torch::Tensor q_weight,
185
- torch::Tensor q_perm,
186
- int bit);
187
-
188
-void static_scaled_fp8_quant(
189
- torch::Tensor& out,
190
- torch::Tensor& input,
191
- torch::Tensor& scale);
192
-
193
-void dynamic_scaled_fp8_quant(
194
- torch::Tensor& out,
195
- torch::Tensor& input,
196
- torch::Tensor& scale);
197
-
198
-void moe_align_block_size(
199
- torch::Tensor topk_ids,
200
- int num_experts,
201
- int block_size,
202
- torch::Tensor sorted_token_ids,
203
- torch::Tensor experts_ids,
204
- torch::Tensor num_tokens_post_pad);
96
+void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
97
+ torch::Tensor lookup_table);
98
+
99
+torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
100
+ torch::Tensor b_gptq_qzeros,
101
+ torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
102
+ bool use_exllama, int bit);
103
+
104
+void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit);
105
+
106
+void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
107
+ torch::Tensor& scale);
108
+
109
+void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor& input,
110
+ torch::Tensor& scale);
111
+
112
+void moe_align_block_size(torch::Tensor topk_ids, int num_experts,
113
+ int block_size, torch::Tensor sorted_token_ids,
114
+ torch::Tensor experts_ids,
115
+ torch::Tensor num_tokens_post_pad);
116
117
#ifndef USE_ROCM
118
using fptr_t = uint64_t;
208
-fptr_t init_custom_ar(torch::Tensor &meta, torch::Tensor &rank_data,
209
- const std::vector<std::string> &handles,
210
- const std::vector<int64_t> &offsets, int rank,
211
- bool full_nvlink);
212
-bool should_custom_ar(torch::Tensor &inp, int max_size, int world_size,
119
+fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
120
+ const std::vector<std::string>& handles,
121
+ const std::vector<int64_t>& offsets, int rank,
122
+ bool full_nvlink);
123
+bool should_custom_ar(torch::Tensor& inp, int max_size, int world_size,
124
bool full_nvlink);
214
-void all_reduce_reg(fptr_t _fa, torch::Tensor &inp, torch::Tensor &out);
215
-void all_reduce_unreg(fptr_t _fa, torch::Tensor &inp, torch::Tensor ®_buffer,
216
- torch::Tensor &out);
125
+void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out);
126
+void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer,
127
+ torch::Tensor& out);
128
void dispose(fptr_t _fa);
129
int meta_size();
219
-void register_buffer(fptr_t _fa, torch::Tensor &t,
220
- const std::vector<std::string> &handles,
221
- const std::vector<int64_t> &offsets);
222
-std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(fptr_t _fa);
223
-void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
224
- const std::vector<std::vector<int64_t>> &offsets);
130
+void register_buffer(fptr_t _fa, torch::Tensor& t,
131
+ const std::vector<std::string>& handles,
132
+ const std::vector<int64_t>& offsets);
133
+std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
134
+ fptr_t _fa);
135
+void register_graph_buffers(fptr_t _fa, const std::vector<std::string>& handles,
136
+ const std::vector<std::vector<int64_t>>& offsets);
137
#endif
csrc/pos_encoding_kernels.cu
+103
-126
@@ -7,14 +7,10 @@
7
8
namespace vllm {
9
10
-template<typename scalar_t, bool IS_NEOX>
10
+template <typename scalar_t, bool IS_NEOX>
11
inline __device__ void apply_token_rotary_embedding(
12
- scalar_t* __restrict__ arr,
13
- const scalar_t* __restrict__ cos_ptr,
14
- const scalar_t* __restrict__ sin_ptr,
15
- int rot_offset,
16
- int embed_dim)
17
-{
12
+ scalar_t* __restrict__ arr, const scalar_t* __restrict__ cos_ptr,
13
+ const scalar_t* __restrict__ sin_ptr, int rot_offset, int embed_dim) {
14
int x_index, y_index;
15
scalar_t cos, sin;
16
if (IS_NEOX) {
@@ -37,19 +33,17 @@ inline __device__ void apply_token_rotary_embedding(
33
arr[y_index] = y * cos + x * sin;
34
}
35
40
-template<typename scalar_t, bool IS_NEOX>
36
+template <typename scalar_t, bool IS_NEOX>
37
inline __device__ void apply_rotary_embedding(
42
- scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
43
- scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
44
- const scalar_t* cache_ptr,
45
- const int head_size,
46
- const int num_heads,
47
- const int num_kv_heads,
48
- const int rot_dim,
49
- const int token_idx,
50
- const int64_t query_stride,
51
- const int64_t key_stride)
52
-{
38
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
39
+ // head_size] or [num_tokens, num_heads,
40
+ // head_size]
41
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
42
+ // head_size] or [num_tokens, num_kv_heads,
43
+ // head_size]
44
+ const scalar_t* cache_ptr, const int head_size, const int num_heads,
45
+ const int num_kv_heads, const int rot_dim, const int token_idx,
46
+ const int64_t query_stride, const int64_t key_stride) {
47
const int embed_dim = rot_dim / 2;
48
const scalar_t* cos_ptr = cache_ptr;
49
const scalar_t* sin_ptr = cache_ptr + embed_dim;
@@ -59,8 +53,8 @@ inline __device__ void apply_rotary_embedding(
53
const int head_idx = i / embed_dim;
54
const int64_t token_head = token_idx * query_stride + head_idx * head_size;
55
const int rot_offset = i % embed_dim;
62
- apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
63
- sin_ptr, rot_offset, embed_dim);
56
+ apply_token_rotary_embedding<scalar_t, IS_NEOX>(
57
+ query + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
58
}
59
60
const int nk = num_kv_heads * embed_dim;
@@ -68,62 +62,74 @@ inline __device__ void apply_rotary_embedding(
62
const int head_idx = i / embed_dim;
63
const int64_t token_head = token_idx * key_stride + head_idx * head_size;
64
const int rot_offset = i % embed_dim;
71
- apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
72
- sin_ptr, rot_offset, embed_dim);
65
+ apply_token_rotary_embedding<scalar_t, IS_NEOX>(
66
+ key + token_head, cos_ptr, sin_ptr, rot_offset, embed_dim);
67
}
68
}
69
76
-template<typename scalar_t, bool IS_NEOX>
70
+template <typename scalar_t, bool IS_NEOX>
71
__global__ void rotary_embedding_kernel(
78
- const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
79
- scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
80
- scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
81
- const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
82
- const int rot_dim,
83
- const int64_t query_stride,
84
- const int64_t key_stride,
85
- const int num_heads,
86
- const int num_kv_heads,
87
- const int head_size) {
72
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or
73
+ // [num_tokens]
74
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
75
+ // head_size] or [num_tokens, num_heads,
76
+ // head_size]
77
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
78
+ // head_size] or [num_tokens, num_kv_heads,
79
+ // head_size]
80
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
81
+ // 2]
82
+ const int rot_dim, const int64_t query_stride, const int64_t key_stride,
83
+ const int num_heads, const int num_kv_heads, const int head_size) {
84
// Each thread block is responsible for one token.
85
const int token_idx = blockIdx.x;
86
int64_t pos = positions[token_idx];
87
const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
88
93
- apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
89
+ apply_rotary_embedding<scalar_t, IS_NEOX>(
90
+ query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim,
91
+ token_idx, query_stride, key_stride);
92
}
93
96
-template<typename scalar_t, bool IS_NEOX>
94
+template <typename scalar_t, bool IS_NEOX>
95
__global__ void batched_rotary_embedding_kernel(
98
- const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99
- scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
100
- scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
101
- const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
102
- const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len] or [num_tokens]
103
- const int rot_dim,
104
- const int64_t query_stride,
105
- const int64_t key_stride,
106
- const int num_heads,
107
- const int num_kv_heads,
108
- const int head_size) {
96
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or
97
+ // [num_tokens]
98
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads,
99
+ // head_size] or [num_tokens, num_heads,
100
+ // head_size]
101
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads,
102
+ // head_size] or [num_tokens, num_kv_heads,
103
+ // head_size]
104
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim //
105
+ // 2]
106
+ const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len]
107
+ // or [num_tokens]
108
+ const int rot_dim, const int64_t query_stride, const int64_t key_stride,
109
+ const int num_heads, const int num_kv_heads, const int head_size) {
110
// Each thread block is responsible for one token.
111
const int token_idx = blockIdx.x;
112
int64_t pos = positions[token_idx];
113
int64_t cos_sin_cache_offset = cos_sin_cache_offsets[token_idx];
113
- const scalar_t* cache_ptr = cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
114
+ const scalar_t* cache_ptr =
115
+ cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
116
115
- apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
117
+ apply_rotary_embedding<scalar_t, IS_NEOX>(
118
+ query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim,
119
+ token_idx, query_stride, key_stride);
120
}
121
118
-} // namespace vllm
122
+} // namespace vllm
123
124
void rotary_embedding(
121
- torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
122
- torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
123
- torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
124
- int head_size,
125
- torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
126
- bool is_neox) {
125
+ torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
126
+ torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
127
+ // [num_tokens, num_heads * head_size]
128
+ torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
129
+ // [num_tokens, num_kv_heads * head_size]
130
+ int head_size,
131
+ torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
132
+ bool is_neox) {
133
int64_t num_tokens = query.numel() / query.size(-1);
134
int rot_dim = cos_sin_cache.size(1);
135
int num_heads = query.size(-1) / head_size;
@@ -135,36 +141,21 @@ void rotary_embedding(
141
dim3 block(std::min(num_heads * rot_dim / 2, 512));
142
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
143
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
138
- VLLM_DISPATCH_FLOATING_TYPES(
139
- query.scalar_type(),
140
- "rotary_embedding",
141
- [&] {
142
- if (is_neox) {
143
- vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
144
- positions.data_ptr<int64_t>(),
145
- query.data_ptr<scalar_t>(),
146
- key.data_ptr<scalar_t>(),
147
- cos_sin_cache.data_ptr<scalar_t>(),
148
- rot_dim,
149
- query_stride,
150
- key_stride,
151
- num_heads,
152
- num_kv_heads,
153
- head_size);
154
- } else {
155
- vllm::rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
156
- positions.data_ptr<int64_t>(),
157
- query.data_ptr<scalar_t>(),
158
- key.data_ptr<scalar_t>(),
159
- cos_sin_cache.data_ptr<scalar_t>(),
160
- rot_dim,
161
- query_stride,
162
- key_stride,
163
- num_heads,
164
- num_kv_heads,
165
- head_size);
166
- }
167
- });
144
+ VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
145
+ if (is_neox) {
146
+ vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
147
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
148
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(), rot_dim,
149
+ query_stride, key_stride, num_heads, num_kv_heads, head_size);
150
+ } else {
151
+ vllm::rotary_embedding_kernel<scalar_t, false>
152
+ <<<grid, block, 0, stream>>>(
153
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
154
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
155
+ rot_dim, query_stride, key_stride, num_heads, num_kv_heads,
156
+ head_size);
157
+ }
158
+ });
159
}
160
161
/*
@@ -172,14 +163,15 @@ Batched version of rotary embedding, pack multiple LoRAs together
163
and process in batched manner.
164
*/
165
void batched_rotary_embedding(
175
- torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
176
- torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
177
- torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
178
- int head_size,
179
- torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
180
- bool is_neox,
181
- int rot_dim,
182
- torch::Tensor& cos_sin_cache_offsets // [num_tokens]
166
+ torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
167
+ torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
168
+ // [num_tokens, num_heads * head_size]
169
+ torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
170
+ // [num_tokens, num_kv_heads * head_size]
171
+ int head_size,
172
+ torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
173
+ bool is_neox, int rot_dim,
174
+ torch::Tensor& cos_sin_cache_offsets // [num_tokens]
175
) {
176
int64_t num_tokens = cos_sin_cache_offsets.size(0);
177
int num_heads = query.size(-1) / head_size;
@@ -191,36 +183,21 @@ void batched_rotary_embedding(
183
dim3 block(std::min(num_heads * rot_dim / 2, 512));
184
const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
185
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
194
- VLLM_DISPATCH_FLOATING_TYPES(
195
- query.scalar_type(),
196
- "rotary_embedding",
197
- [&] {
198
- if (is_neox) {
199
- vllm::batched_rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
200
- positions.data_ptr<int64_t>(),
201
- query.data_ptr<scalar_t>(),
202
- key.data_ptr<scalar_t>(),
203
- cos_sin_cache.data_ptr<scalar_t>(),
204
- cos_sin_cache_offsets.data_ptr<int64_t>(),
205
- rot_dim,
206
- query_stride,
207
- key_stride,
208
- num_heads,
209
- num_kv_heads,
210
- head_size);
211
- } else {
212
- vllm::batched_rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
213
- positions.data_ptr<int64_t>(),
214
- query.data_ptr<scalar_t>(),
215
- key.data_ptr<scalar_t>(),
216
- cos_sin_cache.data_ptr<scalar_t>(),
217
- cos_sin_cache_offsets.data_ptr<int64_t>(),
218
- rot_dim,
219
- query_stride,
220
- key_stride,
221
- num_heads,
222
- num_kv_heads,
223
- head_size);
224
- }
225
- });
186
+ VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "rotary_embedding", [&] {
187
+ if (is_neox) {
188
+ vllm::batched_rotary_embedding_kernel<scalar_t, true>
189
+ <<<grid, block, 0, stream>>>(
190
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
191
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
192
+ cos_sin_cache_offsets.data_ptr<int64_t>(), rot_dim, query_stride,
193
+ key_stride, num_heads, num_kv_heads, head_size);
194
+ } else {
195
+ vllm::batched_rotary_embedding_kernel<scalar_t, false>
196
+ <<<grid, block, 0, stream>>>(
197
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
198
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
199
+ cos_sin_cache_offsets.data_ptr<int64_t>(), rot_dim, query_stride,
200
+ key_stride, num_heads, num_kv_heads, head_size);
201
+ }
202
+ });
203
}
csrc/pybind.cpp
+56
-86
@@ -8,116 +8,87 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8
pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
10
// Attention ops
11
- ops.def(
12
- "paged_attention_v1",
13
- &paged_attention_v1,
14
- "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15
- ops.def(
16
- "paged_attention_v2",
17
- &paged_attention_v2,
18
- "PagedAttention V2.");
11
+ ops.def("paged_attention_v1", &paged_attention_v1,
12
+ "Compute the attention between an input query and the cached "
13
+ "keys/values using PagedAttention.");
14
+ ops.def("paged_attention_v2", &paged_attention_v2, "PagedAttention V2.");
15
16
// Activation ops
21
- ops.def(
22
- "silu_and_mul",
23
- &silu_and_mul,
24
- "Activation function used in SwiGLU.");
25
- ops.def(
26
- "gelu_and_mul",
27
- &gelu_and_mul,
28
- "Activation function used in GeGLU with `none` approximation.");
29
- ops.def(
30
- "gelu_tanh_and_mul",
31
- &gelu_tanh_and_mul,
32
- "Activation function used in GeGLU with `tanh` approximation.");
33
- ops.def(
34
- "gelu_new",
35
- &gelu_new,
36
- "GELU implementation used in GPT-2.");
37
- ops.def(
38
- "gelu_fast",
39
- &gelu_fast,
40
- "Approximate GELU implementation.");
17
+ ops.def("silu_and_mul", &silu_and_mul, "Activation function used in SwiGLU.");
18
+ ops.def("gelu_and_mul", &gelu_and_mul,
19
+ "Activation function used in GeGLU with `none` approximation.");
20
+ ops.def("gelu_tanh_and_mul", &gelu_tanh_and_mul,
21
+ "Activation function used in GeGLU with `tanh` approximation.");
22
+ ops.def("gelu_new", &gelu_new, "GELU implementation used in GPT-2.");
23
+ ops.def("gelu_fast", &gelu_fast, "Approximate GELU implementation.");
24
25
// Layernorm
43
- ops.def(
44
- "rms_norm",
45
- &rms_norm,
46
- "Apply Root Mean Square (RMS) Normalization to the input tensor.");
26
+ ops.def("rms_norm", &rms_norm,
27
+ "Apply Root Mean Square (RMS) Normalization to the input tensor.");
28
48
- ops.def(
49
- "fused_add_rms_norm",
50
- &fused_add_rms_norm,
51
- "In-place fused Add and RMS Normalization");
29
+ ops.def("fused_add_rms_norm", &fused_add_rms_norm,
30
+ "In-place fused Add and RMS Normalization");
31
32
// Rotary embedding
54
- ops.def(
55
- "rotary_embedding",
56
- &rotary_embedding,
57
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
33
+ ops.def("rotary_embedding", &rotary_embedding,
34
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
35
59
- ops.def(
60
- "batched_rotary_embedding",
61
- &batched_rotary_embedding,
62
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key (supports multiple loras)");
36
+ ops.def("batched_rotary_embedding", &batched_rotary_embedding,
37
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key "
38
+ "(supports multiple loras)");
39
40
// Quantization ops
41
#ifndef USE_ROCM
42
ops.def("aqlm_gemm", &aqlm_gemm, "Quantized GEMM for AQLM");
43
ops.def("aqlm_dequant", &aqlm_dequant, "Decompression method for AQLM");
44
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
69
- ops.def("marlin_gemm", &marlin_gemm, "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
70
- ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm, "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
71
- ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72
- ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
45
+ ops.def("marlin_gemm", &marlin_gemm,
46
+ "Marlin (Dense) Optimized Quantized GEMM for GPTQ");
47
+ ops.def("gptq_marlin_24_gemm", &gptq_marlin_24_gemm,
48
+ "Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ");
49
+ ops.def("gptq_marlin_gemm", &gptq_marlin_gemm,
50
+ "gptq_marlin Optimized Quantized GEMM for GPTQ");
51
+ ops.def("gptq_marlin_repack", &gptq_marlin_repack,
52
+ "gptq_marlin repack from GPTQ");
53
ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
74
- ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq, "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column quantization.");
54
+ ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq,
55
+ "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or "
56
+ "per-row/column quantization.");
57
#endif
76
-
58
+
59
ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
60
ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
61
ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
80
- ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant, "Compute FP8 quantized tensor for given scaling factor");
81
- ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant, "Compute FP8 quantized tensor and scaling factor");
82
- ops.def(
83
- "moe_align_block_size",
84
- &moe_align_block_size,
85
- "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
62
+ ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant,
63
+ "Compute FP8 quantized tensor for given scaling factor");
64
+ ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant,
65
+ "Compute FP8 quantized tensor and scaling factor");
66
+ ops.def("moe_align_block_size", &moe_align_block_size,
67
+ "Aligning the number of tokens to be processed by each expert such "
68
+ "that it is divisible by the block size.");
69
70
// Cache ops
71
pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
89
- cache_ops.def(
90
- "swap_blocks",
91
- &swap_blocks,
92
- "Swap in (out) the cache blocks from src to dst");
93
- cache_ops.def(
94
- "copy_blocks",
95
- ©_blocks,
96
- "Copy the cache blocks from src to dst");
97
- cache_ops.def(
98
- "reshape_and_cache",
99
- &reshape_and_cache,
100
- "Reshape the key and value tensors and cache them");
101
- cache_ops.def(
102
- "reshape_and_cache_flash",
103
- &reshape_and_cache_flash,
104
- "Reshape the key and value tensors and cache them");
105
- cache_ops.def(
106
- "convert_fp8",
107
- &convert_fp8,
108
- "Convert the key and value cache to fp8 data type");
72
+ cache_ops.def("swap_blocks", &swap_blocks,
73
+ "Swap in (out) the cache blocks from src to dst");
74
+ cache_ops.def("copy_blocks", ©_blocks,
75
+ "Copy the cache blocks from src to dst");
76
+ cache_ops.def("reshape_and_cache", &reshape_and_cache,
77
+ "Reshape the key and value tensors and cache them");
78
+ cache_ops.def("reshape_and_cache_flash", &reshape_and_cache_flash,
79
+ "Reshape the key and value tensors and cache them");
80
+ cache_ops.def("convert_fp8", &convert_fp8,
81
+ "Convert the key and value cache to fp8 data type");
82
83
// Cuda utils
111
- pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
112
- cuda_utils.def(
113
- "get_device_attribute",
114
- &get_device_attribute,
115
- "Gets the specified device attribute.");
84
+ pybind11::module cuda_utils =
85
+ m.def_submodule("cuda_utils", "vLLM cuda utils");
86
+ cuda_utils.def("get_device_attribute", &get_device_attribute,
87
+ "Gets the specified device attribute.");
88
117
- cuda_utils.def(
118
- "get_max_shared_memory_per_block_device_attribute",
119
- &get_max_shared_memory_per_block_device_attribute,
120
- "Gets the maximum shared memory per block device attribute.");
89
+ cuda_utils.def("get_max_shared_memory_per_block_device_attribute",
90
+ &get_max_shared_memory_per_block_device_attribute,
91
+ "Gets the maximum shared memory per block device attribute.");
92
93
#ifndef USE_ROCM
94
// Custom all-reduce kernels
@@ -134,5 +105,4 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
105
custom_ar.def("register_graph_buffers", ®ister_graph_buffers,
106
"register_graph_buffers");
107
#endif
137
-
108
}
csrc/quantization/aqlm/gemm_kernels.cu
+211
-325
@@ -25,32 +25,28 @@
25
#include <iostream>
26
#include <cstdlib>
27
28
-
28
namespace vllm {
29
namespace aqlm {
30
31
__global__ void Code1x16MatVec(
33
- const int4* __restrict__ A,
34
- const int4* __restrict__ B,
35
- int4* __restrict__ C,
36
- const int4* __restrict__ codebook,
37
- const int prob_m,
38
- const int prob_k,
39
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
40
- const int codebook_stride // as int4.
32
+ const int4* __restrict__ A, const int4* __restrict__ B,
33
+ int4* __restrict__ C, const int4* __restrict__ codebook, const int prob_m,
34
+ const int prob_k,
35
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each
36
+ // codebook, at most 3 long.
37
+ const int codebook_stride // as int4.
38
) {
39
int a_gl_stride = prob_k / 8 / 8;
40
int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
41
bool pred = a_gl_rd < prob_m;
42
46
- if (pred)
47
- {
48
- // advance to the correct codebook, this easy because we only multiply one column of the codebook.
43
+ if (pred) {
44
+ // advance to the correct codebook, this easy because we only multiply one
45
+ // column of the codebook.
46
auto codebook_size = &codebook_a_sizes.x;
50
- while (a_gl_rd >= *codebook_size)
51
- {
52
- codebook += codebook_stride;
53
- ++codebook_size;
47
+ while (a_gl_rd >= *codebook_size) {
48
+ codebook += codebook_stride;
49
+ ++codebook_size;
50
}
51
}
52
@@ -67,8 +63,7 @@ __global__ void Code1x16MatVec(
63
// We pad shared memory to avoid bank conflicts during reads
64
__syncthreads();
65
for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
70
- if (b_gl_rd + i < prob_k / 8)
71
- sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
66
+ if (b_gl_rd + i < prob_k / 8) sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
67
}
68
__syncthreads();
69
b_gl_rd += 32 * 8;
@@ -76,22 +71,19 @@ __global__ void Code1x16MatVec(
71
int b_sh_rd = 9 * (threadIdx.x % 32);
72
if (pred && a_gl_rd < a_gl_end) {
73
const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
79
- #pragma unroll
74
+#pragma unroll
75
for (int i = 0; i < 8; i++) {
76
uint32_t dec[4];
82
- // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
83
- // actually help us; this brings > 2x speedup.
84
- asm volatile (
85
- "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
86
- : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
87
- : "l"((void*) &codebook[enc[i]])
88
- );
77
+ // We bypass the L1 cache to avoid massive amounts of memory streaming
78
+ // that doesn't actually help us; this brings > 2x speedup.
79
+ asm volatile("ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
80
+ : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
81
+ : "l"((void*)&codebook[enc[i]]));
82
half2* a = reinterpret_cast<half2*>(&dec);
83
half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
84
half2 res2 = {};
92
- #pragma unroll
93
- for (int j = 0; j < 4; j++)
94
- res2 = __hfma2(a[j], b[j], res2);
85
+#pragma unroll
86
+ for (int j = 0; j < 4; j++) res2 = __hfma2(a[j], b[j], res2);
87
res += __half2float(res2.x) + __half2float(res2.y);
88
b_sh_rd++;
89
}
@@ -100,37 +92,33 @@ __global__ void Code1x16MatVec(
92
}
93
94
if (pred) {
103
- #pragma unroll
104
- for (int i = 16; i > 0; i /= 2)
105
- res += __shfl_down_sync(0xffffffff, res, i);
95
+#pragma unroll
96
+ for (int i = 16; i > 0; i /= 2) res += __shfl_down_sync(0xffffffff, res, i);
97
if (threadIdx.x % 32 == 0)
98
reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
99
}
100
}
101
102
__global__ void Code2x8MatVec(
112
- const int4* __restrict__ A,
113
- const int4* __restrict__ B,
114
- int4* __restrict__ C,
115
- const int4* __restrict__ codebook,
116
- int prob_m,
117
- int prob_k,
118
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
119
- const int codebook_stride // as int4.
103
+ const int4* __restrict__ A, const int4* __restrict__ B,
104
+ int4* __restrict__ C, const int4* __restrict__ codebook, int prob_m,
105
+ int prob_k,
106
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each
107
+ // codebook, at most 3 long.
108
+ const int codebook_stride // as int4.
109
110
) {
111
int a_gl_stride = prob_k / 8 / 8;
112
int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
113
bool pred = a_gl_rd < prob_m;
114
126
- if (pred)
127
- {
128
- // advance to the correct codebook, this easy because we only multiply one column of the codebook.
115
+ if (pred) {
116
+ // advance to the correct codebook, this easy because we only multiply one
117
+ // column of the codebook.
118
auto codebook_size = &codebook_a_sizes.x;
130
- while (a_gl_rd >= *codebook_size)
131
- {
132
- codebook += codebook_stride;
133
- ++codebook_size;
119
+ while (a_gl_rd >= *codebook_size) {
120
+ codebook += codebook_stride;
121
+ ++codebook_size;
122
}
123
}
124
@@ -148,9 +136,8 @@ __global__ void Code2x8MatVec(
136
137
for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
138
int4 dec = codebook[i];
151
- #pragma unroll
152
- for (int j = 0; j < 8; j++)
153
- sh_code[8 * i + (j + lane) % 8] = dec;
139
+#pragma unroll
140
+ for (int j = 0; j < 8; j++) sh_code[8 * i + (j + lane) % 8] = dec;
141
}
142
__syncthreads();
143
@@ -161,8 +148,7 @@ __global__ void Code2x8MatVec(
148
// We pad shared memory to avoid bank conflicts during reads
149
__syncthreads();
150
for (int i = threadIdx.x; i < 32 * 8; i += blockDim.x) {
164
- if (b_gl_rd + i < prob_k / 8)
165
- sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
151
+ if (b_gl_rd + i < prob_k / 8) sh_b[9 * (i / 8) + i % 8] = B[b_gl_rd + i];
152
}
153
__syncthreads();
154
b_gl_rd += 32 * 8;
@@ -170,13 +156,15 @@ __global__ void Code2x8MatVec(
156
int b_sh_rd = 9 * (threadIdx.x % 32);
157
if (pred && a_gl_rd < a_gl_end) {
158
const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
173
- #pragma unroll
159
+#pragma unroll
160
for (int i = 0; i < 8; i++) {
175
- half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
176
- half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
177
- half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
161
+ half2* a0 =
162
+ reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
163
+ half2* a1 =
164
+ reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
165
+ half2* b = reinterpret_cast<half2*>(&sh_b[b_sh_rd]);
166
half2 res2 = {};
179
- #pragma unroll
167
+#pragma unroll
168
for (int j = 0; j < 4; j++)
169
res2 = __hfma2(__hadd2(a0[j], a1[j]), b[j], res2);
170
res += __half2float(res2.x) + __half2float(res2.y);
@@ -187,36 +175,31 @@ __global__ void Code2x8MatVec(
175
}
176
177
if (pred) {
190
- #pragma unroll
191
- for (int i = 16; i > 0; i /= 2)
192
- res += __shfl_down_sync(0xffffffff, res, i);
178
+#pragma unroll
179
+ for (int i = 16; i > 0; i /= 2) res += __shfl_down_sync(0xffffffff, res, i);
180
if (threadIdx.x % 32 == 0)
181
reinterpret_cast<__half*>(C)[c_gl_wr] = __float2half(res);
182
}
183
}
184
198
-
185
__global__ void Code1x16Dequant(
200
- const int4* __restrict__ A,
201
- int4* __restrict__ C,
202
- const int4* __restrict__ codebook,
203
- int prob_m,
204
- int prob_k,
205
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, sums to m.
206
- const int codebook_stride // as int4
186
+ const int4* __restrict__ A, int4* __restrict__ C,
187
+ const int4* __restrict__ codebook, int prob_m, int prob_k,
188
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each
189
+ // codebook, at most 3 long, sums to m.
190
+ const int codebook_stride // as int4
191
) {
192
int a_gl_stride = prob_k / 8 / 8;
193
int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
194
bool pred = a_gl_rd < prob_m;
195
212
- if (pred)
213
- {
214
- // advance to the correct codebook, this easy because we only multiply one column of the codebook.
196
+ if (pred) {
197
+ // advance to the correct codebook, this easy because we only multiply one
198
+ // column of the codebook.
199
auto codebook_size = &codebook_a_sizes.x;
216
- while (a_gl_rd >= *codebook_size)
217
- {
218
- codebook += codebook_stride;
219
- ++codebook_size;
200
+ while (a_gl_rd >= *codebook_size) {
201
+ codebook += codebook_stride;
202
+ ++codebook_size;
203
}
204
}
205
@@ -231,17 +214,15 @@ __global__ void Code1x16Dequant(
214
while (iters--) {
215
if (pred && a_gl_rd < a_gl_end) {
216
const uint16_t* enc = reinterpret_cast<const uint16_t*>(&A[a_gl_rd]);
234
- #pragma unroll
217
+#pragma unroll
218
for (int i = 0; i < 8; i++) {
219
int4 chunk;
220
auto dec = reinterpret_cast<uint32_t*>(&chunk);
238
- // We bypass the L1 cache to avoid massive amounts of memory streaming that doesn't
239
- // actually help us; this brings > 2x speedup.
240
- asm volatile (
241
- "ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
242
- : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
243
- : "l"((void*) &codebook[enc[i]])
244
- );
221
+ // We bypass the L1 cache to avoid massive amounts of memory streaming
222
+ // that doesn't actually help us; this brings > 2x speedup.
223
+ asm volatile("ld.cg.global.v4.u32 {%0, %1, %2, %3}, [%4];"
224
+ : "=r"(dec[0]), "=r"(dec[1]), "=r"(dec[2]), "=r"(dec[3])
225
+ : "l"((void*)&codebook[enc[i]]));
226
227
C[a_gl_rd * 8 + i] = chunk;
228
}
@@ -250,28 +231,25 @@ __global__ void Code1x16Dequant(
231
}
232
}
233
253
-
234
__global__ void Code2x8Dequant(
255
- const int4* __restrict__ A,
256
- int4* __restrict__ C,
257
- const int4* __restrict__ codebook,
258
- int prob_m,
259
- int prob_k,
260
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
261
- const int codebook_stride // as int4
235
+ const int4* __restrict__ A, int4* __restrict__ C,
236
+ const int4* __restrict__ codebook, int prob_m, int prob_k,
237
+ const int4
238
+ codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
239
+ // most 3 long, corresponds to cols.
240
+ const int codebook_stride // as int4
241
) {
242
int a_gl_stride = prob_k / 8 / 8;
243
int a_gl_rd = (blockDim.x / 32) * blockIdx.x + (threadIdx.x / 32);
244
bool pred = a_gl_rd < prob_m;
245
267
- if (pred)
268
- {
269
- // advance to the correct codebook, this easy because we only multiply one column of the codebook.
246
+ if (pred) {
247
+ // advance to the correct codebook, this easy because we only multiply one
248
+ // column of the codebook.
249
auto codebook_size = &codebook_a_sizes.x;
271
- while (a_gl_rd >= *codebook_size)
272
- {
273
- codebook += codebook_stride;
274
- ++codebook_size;
250
+ while (a_gl_rd >= *codebook_size) {
251
+ codebook += codebook_stride;
252
+ ++codebook_size;
253
}
254
}
255
@@ -290,9 +268,8 @@ __global__ void Code2x8Dequant(
268
269
for (int i = threadIdx.x; i < 2 * 256; i += blockDim.x) {
270
int4 dec = codebook[i];
293
- #pragma unroll
294
- for (int j = 0; j < 8; j++)
295
- sh_code[8 * i + (j + lane) % 8] = dec;
271
+#pragma unroll
272
+ for (int j = 0; j < 8; j++) sh_code[8 * i + (j + lane) % 8] = dec;
273
}
274
__syncthreads();
275
@@ -302,12 +279,14 @@ __global__ void Code2x8Dequant(
279
while (iters--) {
280
if (pred && a_gl_rd < a_gl_end) {
281
const uint8_t* enc = reinterpret_cast<const uint8_t*>(&A[a_gl_rd]);
305
- #pragma unroll
282
+#pragma unroll
283
for (int i = 0; i < 8; i++) {
284
int4 chunk;
308
- half2* a0 = reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
309
- half2* a1 = reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
310
- #pragma unroll
285
+ half2* a0 =
286
+ reinterpret_cast<half2*>(&sh_code0[8 * enc[2 * i + 0] + lane]);
287
+ half2* a1 =
288
+ reinterpret_cast<half2*>(&sh_code1[8 * enc[2 * i + 1] + lane]);
289
+#pragma unroll
290
for (int j = 0; j < 4; j++)
291
reinterpret_cast<half2*>(&chunk)[j] = __hadd2(a0[j], a1[j]);
292
C[a_gl_rd * 8 + i] = chunk;
@@ -317,22 +296,15 @@ __global__ void Code2x8Dequant(
296
}
297
}
298
320
-inline int ceildiv(int a, int b) {
321
- return (a + b - 1) / b;
322
-}
299
+inline int ceildiv(int a, int b) { return (a + b - 1) / b; }
300
301
const int THREAD_M = 16;
302
326
-void code1x16_matvec_cuda(
327
- const void* __restrict__ A,
328
- const void* __restrict__ B,
329
- void* __restrict__ C,
330
- const void* __restrict__ codebook,
331
- int prob_m,
332
- int prob_k,
333
- const int4 codebook_a_sizes,
334
- const int codebook_stride
335
-) {
303
+void code1x16_matvec_cuda(const void* __restrict__ A,
304
+ const void* __restrict__ B, void* __restrict__ C,
305
+ const void* __restrict__ codebook, int prob_m,
306
+ int prob_k, const int4 codebook_a_sizes,
307
+ const int codebook_stride) {
308
int sms;
309
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
310
int waves = 0;
@@ -345,28 +317,16 @@ void code1x16_matvec_cuda(
317
int blocks = ceildiv(prob_m, thread_m);
318
int threads = 32 * thread_m;
319
cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
348
- Code1x16MatVec<<<blocks, threads, 16*32*9, stream>>>(
349
- (const int4*) A,
350
- (const int4*) B,
351
- (int4*) C,
352
- (const int4*) codebook,
353
- prob_m,
354
- prob_k,
355
- codebook_a_sizes,
356
- codebook_stride
357
- );
320
+ Code1x16MatVec<<<blocks, threads, 16 * 32 * 9, stream>>>(
321
+ (const int4*)A, (const int4*)B, (int4*)C, (const int4*)codebook, prob_m,
322
+ prob_k, codebook_a_sizes, codebook_stride);
323
}
324
360
-void code2x8_matvec_cuda(
361
- const void* __restrict__ A,
362
- const void* __restrict__ B,
363
- void* __restrict__ C,
364
- const void* __restrict__ codebook,
365
- int prob_m,
366
- int prob_k,
367
- const int4 codebook_a_sizes,
368
- const int codebook_stride
369
-) {
325
+void code2x8_matvec_cuda(const void* __restrict__ A, const void* __restrict__ B,
326
+ void* __restrict__ C,
327
+ const void* __restrict__ codebook, int prob_m,
328
+ int prob_k, const int4 codebook_a_sizes,
329
+ const int codebook_stride) {
330
int sms;
331
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
332
int waves = 0;
@@ -379,30 +339,20 @@ void code2x8_matvec_cuda(
339
int blocks = ceildiv(prob_m, thread_m);
340
int threads = 32 * thread_m;
341
int shared = 16 * (2 * 256 * 8 + 32 * 9);
382
- cudaFuncSetAttribute(
383
- Code2x8MatVec, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
384
- );
342
+ cudaFuncSetAttribute(Code2x8MatVec,
343
+ cudaFuncAttributeMaxDynamicSharedMemorySize, shared);
344
cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
345
Code2x8MatVec<<<blocks, threads, shared, stream>>>(
387
- (const int4*) A,
388
- (const int4*) B,
389
- (int4*) C,
390
- (const int4*) codebook,
391
- prob_m,
392
- prob_k,
393
- codebook_a_sizes,
394
- codebook_stride
395
- );
346
+ (const int4*)A, (const int4*)B, (int4*)C, (const int4*)codebook, prob_m,
347
+ prob_k, codebook_a_sizes, codebook_stride);
348
}
349
350
void code1x16_dequant_cuda(
399
- const void* __restrict__ A,
400
- void* __restrict__ C,
401
- const void* __restrict__ codebook,
402
- int prob_m,
403
- int prob_k,
404
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
405
- const int codebook_stride // as int4.
351
+ const void* __restrict__ A, void* __restrict__ C,
352
+ const void* __restrict__ codebook, int prob_m, int prob_k,
353
+ const int4 codebook_a_sizes, // cumulative sizes of A spanning each
354
+ // codebook, at most 3 long.
355
+ const int codebook_stride // as int4.
356
) {
357
int sms;
358
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
@@ -417,25 +367,21 @@ void code1x16_dequant_cuda(
367
int threads = 32 * thread_m;
368
cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
369
Code1x16Dequant<<<blocks, threads, 0, stream>>>(
420
- (const int4*) A,
421
- (int4*) C,
422
- (const int4*) codebook,
423
- prob_m,
424
- prob_k,
425
- codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long.
426
- codebook_stride // as int4.
370
+ (const int4*)A, (int4*)C, (const int4*)codebook, prob_m, prob_k,
371
+ codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
372
+ // most 3 long.
373
+ codebook_stride // as int4.
374
);
375
}
376
377
// Dequantizes the code and codebook into weights.
431
-void code2x8_dequant_cuda(
432
- const void* __restrict__ A,
433
- void* __restrict__ C,
434
- const void* __restrict__ codebook,
435
- int prob_m,
436
- int prob_k,
437
- const int4 codebook_a_sizes, // cumulative sizes of A spanning each codebook, at most 3 long, corresponds to cols.
438
- const int codebook_stride // as int4
378
+void code2x8_dequant_cuda(
379
+ const void* __restrict__ A, void* __restrict__ C,
380
+ const void* __restrict__ codebook, int prob_m, int prob_k,
381
+ const int4
382
+ codebook_a_sizes, // cumulative sizes of A spanning each codebook, at
383
+ // most 3 long, corresponds to cols.
384
+ const int codebook_stride // as int4
385
) {
386
int sms;
387
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, 0);
@@ -451,74 +397,50 @@ void code2x8_dequant_cuda(
397
int shared = 16 * (2 * 256 * 8 + 32 * 9);
398
cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
399
454
- cudaFuncSetAttribute(
455
- Code2x8Dequant, cudaFuncAttributeMaxDynamicSharedMemorySize, shared
456
- );
400
+ cudaFuncSetAttribute(Code2x8Dequant,
401
+ cudaFuncAttributeMaxDynamicSharedMemorySize, shared);
402
Code2x8Dequant<<<blocks, threads, shared, stream>>>(
458
- (const int4*) A,
459
- (int4*) C,
460
- (const int4*) codebook,
461
- prob_m,
462
- prob_k,
463
- codebook_a_sizes,
464
- codebook_stride
465
- );
403
+ (const int4*)A, (int4*)C, (const int4*)codebook, prob_m, prob_k,
404
+ codebook_a_sizes, codebook_stride);
405
}
406
468
-int codebook_stride(const torch::Tensor& codebooks)
469
-{
407
+int codebook_stride(const torch::Tensor& codebooks) {
408
return codebooks.stride(0) * codebooks.element_size() / sizeof(int4);
409
}
410
411
void code1x16_matvec(
474
- const torch::Tensor& A,
475
- const torch::Tensor& B,
476
- torch::Tensor& C,
477
- const torch::Tensor& codebook,
478
- const int4 codebook_a_sizes // cumulative sizes of A spanning each codebook, at most 3 long.
412
+ const torch::Tensor& A, const torch::Tensor& B, torch::Tensor& C,
413
+ const torch::Tensor& codebook,
414
+ const int4 codebook_a_sizes // cumulative sizes of A spanning each
415
+ // codebook, at most 3 long.
416
) {
417
const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
418
int prob_m = C.size(0);
419
int prob_k = B.size(0);
420
484
- code1x16_matvec_cuda(
485
- A.data_ptr(),
486
- B.data_ptr(),
487
- C.data_ptr(),
488
- codebook.data_ptr(),
489
- prob_m,
490
- prob_k,
491
- codebook_a_sizes,
492
- codebook_stride(codebook)
493
- );
421
+ code1x16_matvec_cuda(A.data_ptr(), B.data_ptr(), C.data_ptr(),
422
+ codebook.data_ptr(), prob_m, prob_k, codebook_a_sizes,
423
+ codebook_stride(codebook));
424
}
425
496
-torch::Tensor code1x16_matmat(
497
- const torch::Tensor& input,
498
- const torch::Tensor& codes,
499
- const torch::Tensor& codebooks,
500
- const torch::Tensor& scales,
501
- const int4 codebook_a_sizes,
502
- const std::optional<torch::Tensor>& bias) {
426
+torch::Tensor code1x16_matmat(const torch::Tensor& input,
427
+ const torch::Tensor& codes,
428
+ const torch::Tensor& codebooks,
429
+ const torch::Tensor& scales,
430
+ const int4 codebook_a_sizes,
431
+ const std::optional<torch::Tensor>& bias) {
432
auto input_sizes = input.sizes();
433
auto out_features = codes.size(0) * codebooks.size(2);
434
auto flat_input = input.reshape({-1, input.size(-1)});
506
- auto flat_output = torch::empty({flat_input.size(0), out_features},
507
- torch::TensorOptions()
508
- .dtype(input.dtype())
509
- .device(input.device())
510
- );
435
+ auto flat_output = torch::empty(
436
+ {flat_input.size(0), out_features},
437
+ torch::TensorOptions().dtype(input.dtype()).device(input.device()));
438
439
for (int i = 0; i < flat_input.size(0); ++i) {
440
auto input_vec = flat_input.index({i});
441
auto output_vec = flat_output.index({i});
515
- code1x16_matvec(
516
- codes.squeeze(2),
517
- input_vec,
518
- output_vec,
519
- codebooks,
520
- codebook_a_sizes
521
- );
442
+ code1x16_matvec(codes.squeeze(2), input_vec, output_vec, codebooks,
443
+ codebook_a_sizes);
444
}
445
flat_output *= scales.flatten().unsqueeze(0);
446
@@ -533,55 +455,35 @@ torch::Tensor code1x16_matmat(
455
return output;
456
}
457
536
-void code2x8_matvec(
537
- const torch::Tensor& A,
538
- const torch::Tensor& B,
539
- torch::Tensor& C,
540
- const torch::Tensor& codebook,
541
- const int4 codebook_a_sizes
542
-) {
458
+void code2x8_matvec(const torch::Tensor& A, const torch::Tensor& B,
459
+ torch::Tensor& C, const torch::Tensor& codebook,
460
+ const int4 codebook_a_sizes) {
461
const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
462
int prob_m = C.size(0);
463
int prob_k = B.size(0);
546
- code2x8_matvec_cuda(
547
- A.data_ptr(),
548
- B.data_ptr(),
549
- C.data_ptr(),
550
- codebook.data_ptr(),
551
- prob_m,
552
- prob_k,
553
- codebook_a_sizes,
554
- 2 * codebook_stride(codebook)
555
- );
464
+ code2x8_matvec_cuda(A.data_ptr(), B.data_ptr(), C.data_ptr(),
465
+ codebook.data_ptr(), prob_m, prob_k, codebook_a_sizes,
466
+ 2 * codebook_stride(codebook));
467
}
468
558
-torch::Tensor code2x8_matmat(
559
- const torch::Tensor& input,
560
- const torch::Tensor& codes,
561
- const torch::Tensor& codebooks,
562
- const torch::Tensor& scales,
563
- const int4 codebook_a_sizes,
564
- const std::optional<torch::Tensor>& bias
565
-) {
469
+torch::Tensor code2x8_matmat(const torch::Tensor& input,
470
+ const torch::Tensor& codes,
471
+ const torch::Tensor& codebooks,
472
+ const torch::Tensor& scales,
473
+ const int4 codebook_a_sizes,
474
+ const std::optional<torch::Tensor>& bias) {
475
auto input_sizes = input.sizes();
476
auto out_features = codes.size(0) * codebooks.size(2);
477
auto flat_input = input.reshape({-1, input.size(-1)});
569
- auto flat_output = torch::empty({flat_input.size(0), out_features},
570
- torch::TensorOptions()
571
- .dtype(input.dtype())
572
- .device(input.device())
573
- );
478
+ auto flat_output = torch::empty(
479
+ {flat_input.size(0), out_features},
480
+ torch::TensorOptions().dtype(input.dtype()).device(input.device()));
481
482
for (int i = 0; i < flat_input.size(0); ++i) {
483
auto input_vec = flat_input.index({i});
484
auto output_vec = flat_output.index({i});
578
- code2x8_matvec(
579
- codes.squeeze(2),
580
- input_vec,
581
- output_vec,
582
- codebooks,
583
- codebook_a_sizes
584
- );
485
+ code2x8_matvec(codes.squeeze(2), input_vec, output_vec, codebooks,
486
+ codebook_a_sizes);
487
}
488
flat_output *= scales.flatten().unsqueeze(0);
489
if (bias.has_value()) {
@@ -596,64 +498,56 @@ torch::Tensor code2x8_matmat(
498
}
499
500
// Accumulate the partition sizes.
599
-int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes)
600
-{
501
+int4 accumulate_sizes(const torch::Tensor& codebook_partition_sizes) {
502
int4 cumulative_sizes;
503
auto cumulative_size = &cumulative_sizes.x;
504
int i = 0;
505
int last = 0;
506
assert(codebook_partition_sizes.size(0) <= 4);
606
- for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size)
607
- {
507
+ for (; i < codebook_partition_sizes.size(0); ++i, ++cumulative_size) {
508
*cumulative_size = codebook_partition_sizes[i].item<int>() + last;
509
last = *cumulative_size;
510
}
511
// fill in the rest with unreachable.
612
- for (; i < 4; ++i, ++cumulative_size)
613
- {
614
- *cumulative_size = last*10;
512
+ for (; i < 4; ++i, ++cumulative_size) {
513
+ *cumulative_size = last * 10;
514
}
515
return cumulative_sizes;
516
}
517
619
-} // namespace aqlm
620
-} // namespace vllm
621
-
518
+} // namespace aqlm
519
+} // namespace vllm
520
623
-torch::Tensor aqlm_gemm(
624
- const torch::Tensor& input,
625
- const torch::Tensor& codes,
626
- const torch::Tensor& codebooks,
627
- const torch::Tensor& scales,
628
- const torch::Tensor& codebook_partition_sizes,
629
- const std::optional<torch::Tensor>& bias
630
-)
631
-{
632
- int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
521
+torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
522
+ const torch::Tensor& codebooks,
523
+ const torch::Tensor& scales,
524
+ const torch::Tensor& codebook_partition_sizes,
525
+ const std::optional<torch::Tensor>& bias) {
526
+ int4 cumulative_sizes =
527
+ vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
528
529
int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
530
int const entries = codebooks.size(1);
531
637
- if (nbooks == 1 && entries == (1 << 16))
638
- {
639
- return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
532
+ if (nbooks == 1 && entries == (1 << 16)) {
533
+ return vllm::aqlm::code1x16_matmat(input, codes, codebooks, scales,
534
+ cumulative_sizes, bias);
535
}
641
- if (nbooks == 2 && entries == (1 << 8))
642
- {
643
- return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales, cumulative_sizes, bias);
536
+ if (nbooks == 2 && entries == (1 << 8)) {
537
+ return vllm::aqlm::code2x8_matmat(input, codes, codebooks, scales,
538
+ cumulative_sizes, bias);
539
}
540
646
- TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
541
+ TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries,
542
+ " entries is not currently supported.")
543
return {};
544
}
545
650
-torch::Tensor aqlm_dequant(
651
- const torch::Tensor& codes,
652
- const torch::Tensor& codebooks,
653
- const torch::Tensor& codebook_partition_sizes
654
-)
655
-{
656
- int4 cumulative_sizes = vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
546
+torch::Tensor aqlm_dequant(const torch::Tensor& codes,
547
+ const torch::Tensor& codebooks,
548
+ const torch::Tensor& codebook_partition_sizes) {
549
+ int4 cumulative_sizes =
550
+ vllm::aqlm::accumulate_sizes(codebook_partition_sizes);
551
552
int const nbooks = codebooks.size(0) / codebook_partition_sizes.size(0);
553
int const entries = codebooks.size(1);
@@ -668,45 +562,37 @@ torch::Tensor aqlm_dequant(
562
assert(out_features = codebook_partition_sizes.sum().item<int>());
563
564
auto weights = torch::empty({out_features, in_features},
671
- torch::TensorOptions()
672
- .dtype(codebooks.dtype())
673
- .device(codebooks.device())
674
- );
565
+ torch::TensorOptions()
566
+ .dtype(codebooks.dtype())
567
+ .device(codebooks.device()));
568
+
569
+ if (nbooks == 1 && entries == (1 << 16)) {
570
+ vllm::aqlm::code1x16_dequant_cuda(codes.data_ptr(), weights.data_ptr(),
571
+ codebooks.data_ptr(), out_features,
572
+ in_features, cumulative_sizes,
573
+ vllm::aqlm::codebook_stride(codebooks));
574
676
- if (nbooks == 1 && entries == (1 << 16))
677
- {
678
- vllm::aqlm::code1x16_dequant_cuda(
679
- codes.data_ptr(),
680
- weights.data_ptr(),
681
- codebooks.data_ptr(),
682
- out_features,
683
- in_features,
684
- cumulative_sizes,
685
- vllm::aqlm::codebook_stride(codebooks));
686
-
687
- // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation.)
688
- // weights *= scales.index({"...", 0, 0});
689
-
690
- return weights;
575
+ // if you wanted to flip to scaling the weights, (though it's 30%-ish slower
576
+ // and not consistent with gemv implementation.) weights *=
577
+ // scales.index({"...", 0, 0});
578
+
579
+ return weights;
580
}
581
693
- if (nbooks == 2 && entries == (1 << 8))
694
- {
695
- vllm::aqlm::code2x8_dequant_cuda(
696
- codes.data_ptr(),
697
- weights.data_ptr(),
698
- codebooks.data_ptr(),
699
- out_features,
700
- in_features,
701
- cumulative_sizes,
702
- vllm::aqlm::codebook_stride(codebooks));
703
-
704
- // if you wanted to flip to scaling the weights, (though it's 30%-ish slower and not consistent with gemv implementation)
705
- // weights *= scales.index({"...", 0, 0});
706
-
707
- return weights;
582
+ if (nbooks == 2 && entries == (1 << 8)) {
583
+ vllm::aqlm::code2x8_dequant_cuda(codes.data_ptr(), weights.data_ptr(),
584
+ codebooks.data_ptr(), out_features,
585
+ in_features, cumulative_sizes,
586
+ vllm::aqlm::codebook_stride(codebooks));
587
+
588
+ // if you wanted to flip to scaling the weights, (though it's 30%-ish slower
589
+ // and not consistent with gemv implementation) weights *=
590
+ // scales.index({"...", 0, 0});
591
+
592
+ return weights;
593
}
594
710
- TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries, " entries is not currently supported.")
595
+ TORCH_CHECK(false, "AQLM with ", nbooks, " codebooks and ", entries,
596
+ " entries is not currently supported.")
597
return {};
598
}
csrc/quantization/awq/dequantize.cuh
+76
-62
@@ -1,11 +1,11 @@
1
/*
2
Adapted from https://github.com/mit-han-lab/llm-awq
3
-Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
3
+Modified from NVIDIA FasterTransformer:
4
+https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
5
@article{lin2023awq,
5
- title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
6
- author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
7
- journal={arXiv},
8
- year={2023}
6
+ title={AWQ: Activation-aware Weight Quantization for LLM Compression and
7
+Acceleration}, author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang,
8
+Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
9
}
10
*/
11
@@ -14,74 +14,88 @@ Modified from NVIDIA FasterTransformer: https://github.com/NVIDIA/FasterTransfor
14
namespace vllm {
15
namespace awq {
16
17
-__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source)
18
-{
17
+__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source) {
18
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
19
assert(false);
20
#else
22
- uint4 result;
21
+ uint4 result;
22
24
- uint32_t* h = reinterpret_cast<uint32_t*>(&result);
25
- uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
23
+ uint32_t* h = reinterpret_cast<uint32_t*>(&result);
24
+ uint32_t const i4s = reinterpret_cast<uint32_t const&>(source);
25
27
- // First, we extract the i4s and construct an intermediate fp16 number.
28
- static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
29
- static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
30
- static constexpr uint32_t TOP_MASK = 0x00f000f0;
31
- static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
26
+ // First, we extract the i4s and construct an intermediate fp16 number.
27
+ static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa;
28
+ static constexpr uint32_t BOTTOM_MASK = 0x000f000f;
29
+ static constexpr uint32_t TOP_MASK = 0x00f000f0;
30
+ static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400;
31
33
- // Note that the entire sequence only requires 1 shift instruction. This is thanks to the register packing
34
- // format and the fact that we force our integers to be unsigned, and account for this in the fp16 subtractions.
35
- // In addition, I exploit the fact that sub and fma have the same throughput in order to convert elt_23 and
36
- // elt_67 to fp16 without having to shift them to the bottom bits before hand.
32
+ // Note that the entire sequence only requires 1 shift instruction. This is
33
+ // thanks to the register packing format and the fact that we force our
34
+ // integers to be unsigned, and account for this in the fp16 subtractions. In
35
+ // addition, I exploit the fact that sub and fma have the same throughput in
36
+ // order to convert elt_23 and elt_67 to fp16 without having to shift them to
37
+ // the bottom bits before hand.
38
38
- // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW dependency if we issue
39
- // immediately before required.
40
- const uint32_t top_i4s = i4s >> 8;
41
- // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
42
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
43
- : "=r"(h[0])
44
- : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
45
- // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
46
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
47
- : "=r"(h[1])
48
- : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
49
- // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
50
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
51
- : "=r"(h[2])
52
- : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
53
- // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
54
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
55
- : "=r"(h[3])
56
- : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut));
39
+ // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW
40
+ // dependency if we issue immediately before required.
41
+ const uint32_t top_i4s = i4s >> 8;
42
+ // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400
43
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
44
+ : "=r"(h[0])
45
+ : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
46
+ "n"(immLut));
47
+ // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400
48
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
49
+ : "=r"(h[1])
50
+ : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
51
+ "n"(immLut));
52
+ // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400
53
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
54
+ : "=r"(h[2])
55
+ : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
56
+ "n"(immLut));
57
+ // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400
58
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
59
+ : "=r"(h[3])
60
+ : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM),
61
+ "n"(immLut));
62
58
- // I use inline PTX below because I am not sure if the compiler will emit float2half instructions if I use the
59
- // half2 ctor. In this case, I chose performance reliability over code readability.
63
+ // I use inline PTX below because I am not sure if the compiler will emit
64
+ // float2half instructions if I use the half2 ctor. In this case, I chose
65
+ // performance reliability over code readability.
66
61
- // This is the half2 {1032, 1032} represented as an integer.
62
- // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
63
- // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
64
- static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
65
- // This is the half2 {1 / 16, 1 / 16} represented as an integer.
66
- static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
67
- // This is the half2 {-72, -72} represented as an integer.
68
- // static constexpr uint32_t NEG_72 = 0xd480d480;
69
- // Haotian: Let's use {-64, -64}.
70
- static constexpr uint32_t NEG_64 = 0xd400d400;
67
+ // This is the half2 {1032, 1032} represented as an integer.
68
+ // static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408;
69
+ // Haotian: subtract {1024, 1024} instead, we do not need to map to [-8, 7]
70
+ static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400;
71
+ // This is the half2 {1 / 16, 1 / 16} represented as an integer.
72
+ static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00;
73
+ // This is the half2 {-72, -72} represented as an integer.
74
+ // static constexpr uint32_t NEG_72 = 0xd480d480;
75
+ // Haotian: Let's use {-64, -64}.
76
+ static constexpr uint32_t NEG_64 = 0xd400d400;
77
72
- // Finally, we construct the output numbers.
73
- // Convert elt_01
74
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
75
- // Convert elt_23
76
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
77
- // Convert elt_45
78
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
79
- // Convert elt_67
80
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
78
+ // Finally, we construct the output numbers.
79
+ // Convert elt_01
80
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
81
+ : "=r"(h[0])
82
+ : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM));
83
+ // Convert elt_23
84
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
85
+ : "=r"(h[1])
86
+ : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
87
+ // Convert elt_45
88
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
89
+ : "=r"(h[2])
90
+ : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM));
91
+ // Convert elt_67
92
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
93
+ : "=r"(h[3])
94
+ : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64));
95
82
- return result;
96
+ return result;
97
#endif
98
}
99
86
-} // namespace awq
87
-} // namespace vllm
100
+} // namespace awq
101
+} // namespace vllm
csrc/quantization/awq/gemm_kernels.cu
+357
-254
@@ -1,14 +1,12 @@
1
/*
2
Adapted from https://github.com/mit-han-lab/llm-awq
3
@article{lin2023awq,
4
- title={AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration},
5
- author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang, Shang and Dang, Xingyu and Han, Song},
6
- journal={arXiv},
7
- year={2023}
4
+ title={AWQ: Activation-aware Weight Quantization for LLM Compression and
5
+Acceleration}, author={Lin, Ji and Tang, Jiaming and Tang, Haotian and Yang,
6
+Shang and Dang, Xingyu and Han, Song}, journal={arXiv}, year={2023}
7
}
8
*/
9
11
-
10
#include <torch/extension.h>
11
#include <c10/cuda/CUDAGuard.h>
12
@@ -20,26 +18,20 @@ namespace vllm {
18
namespace awq {
19
20
// Pack two half values.
23
-static inline __device__ __host__ unsigned
24
-__pack_half2(const half x, const half y) {
25
- unsigned v0 = *((unsigned short *)&x);
26
- unsigned v1 = *((unsigned short *)&y);
21
+static inline __device__ __host__ unsigned __pack_half2(const half x,
22
+ const half y) {
23
+ unsigned v0 = *((unsigned short*)&x);
24
+ unsigned v1 = *((unsigned short*)&y);
25
return (v1 << 16) | v0;
26
}
27
30
-template<int N>
31
-__global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
32
- int G,
33
- int split_k_iters,
34
- half* __restrict__ A,
35
- int* __restrict__ B,
36
- half* __restrict__ scaling_factors,
37
- int* __restrict__ zeros,
38
- int M,
39
- int IC,
40
- int OC,
41
- half* __restrict__ C)
42
-{
28
+template <int N>
29
+__global__ void __launch_bounds__(64)
30
+ gemm_forward_4bit_cuda_m16nXk32(int G, int split_k_iters,
31
+ half* __restrict__ A, int* __restrict__ B,
32
+ half* __restrict__ scaling_factors,
33
+ int* __restrict__ zeros, int M, int IC,
34
+ int OC, half* __restrict__ C) {
35
// Only support matrix n = 64 or 128
36
assert(N == 64 || N == 128);
37
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
@@ -70,43 +62,46 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
62
static constexpr int row_stride = 2 * 32 * 8 / N;
63
bool ld_zero_flag = (threadIdx.y * 32 + threadIdx.x) * 8 < N;
64
// TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
73
- bool ld_A_flag = (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp + threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
65
+ bool ld_A_flag =
66
+ (blockIdx_y / j_factors1 * 16 + threadIdx.y * row_stride_warp +
67
+ threadIdx.x * 8 / 32) < M; // threadIdx.y is warp_id
68
// bool wb_C_flag = (threadIdx.x / 4) < M;
69
76
- half* A_ptr = A
77
- + (((int)blockIdx_y) / j_factors1 * 16 + (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) * IC
78
- + (((int)threadIdx.x) % (32 / 8)) * 8;
79
-
80
- int* B_ptr = B
81
- + ((int)threadIdx.y) * (OC / 8) * (256 / N)
82
- + (((int)threadIdx.x) / (N / 8)) * (OC / 8)
83
- + (((int)blockIdx_y) % j_factors1) * (N / 8)
84
- + (((int)threadIdx.x) % (N / 8)) * 1;
85
-// Why * 1 in the above line?
86
-
87
- half* A_shared_ptr = A_shared
88
- + ((int)threadIdx.y) * row_stride_warp * (32 + 8)
89
- + (((int)threadIdx.x) / (32 / 8)) * (32 + 8)
90
- + (((int)threadIdx.x) % (32 / 8) ) * 8;
91
-
92
- half* B_shared_ptr = B_shared
93
- + ((int)threadIdx.y) * (row_stride / 2) * (N + 8)
94
- + (((int)threadIdx.x) / (N / 8)) * (N + 8)
95
- + (((int)threadIdx.x) % (N / 8)) * 8;
96
-
97
- int* zeros_ptr = zeros
98
- + (((int)blockIdx_y) % j_factors1) * (N / 8)
99
- + ((int)threadIdx.x) % (N / 8);
100
-
101
- half* scaling_factors_ptr = scaling_factors
102
- + (((int)blockIdx_y) % j_factors1) * N
103
- + (((int)threadIdx.x) % (N / 8)) * 8;
104
-
105
- half* C_ptr = C
106
- + static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
107
- + (((int)blockIdx_y) % j_factors1) * N
108
- + ((int)threadIdx.y) * (N / 2)
109
- + (((int)threadIdx.x) % 4) * 2;
70
+ half* A_ptr =
71
+ A +
72
+ (((int)blockIdx_y) / j_factors1 * 16 +
73
+ (((int)threadIdx.y) * row_stride_warp) + ((int)threadIdx.x) / (32 / 8)) *
74
+ IC +
75
+ (((int)threadIdx.x) % (32 / 8)) * 8;
76
+
77
+ int* B_ptr = B + ((int)threadIdx.y) * (OC / 8) * (256 / N) +
78
+ (((int)threadIdx.x) / (N / 8)) * (OC / 8) +
79
+ (((int)blockIdx_y) % j_factors1) * (N / 8) +
80
+ (((int)threadIdx.x) % (N / 8)) * 1;
81
+ // Why * 1 in the above line?
82
+
83
+ half* A_shared_ptr = A_shared +
84
+ ((int)threadIdx.y) * row_stride_warp * (32 + 8) +
85
+ (((int)threadIdx.x) / (32 / 8)) * (32 + 8) +
86
+ (((int)threadIdx.x) % (32 / 8)) * 8;
87
+
88
+ half* B_shared_ptr = B_shared +
89
+ ((int)threadIdx.y) * (row_stride / 2) * (N + 8) +
90
+ (((int)threadIdx.x) / (N / 8)) * (N + 8) +
91
+ (((int)threadIdx.x) % (N / 8)) * 8;
92
+
93
+ int* zeros_ptr = zeros + (((int)blockIdx_y) % j_factors1) * (N / 8) +
94
+ ((int)threadIdx.x) % (N / 8);
95
+
96
+ half* scaling_factors_ptr = scaling_factors +
97
+ (((int)blockIdx_y) % j_factors1) * N +
98
+ (((int)threadIdx.x) % (N / 8)) * 8;
99
+
100
+ half* C_ptr =
101
+ C +
102
+ static_cast<long long>(blockIdx_z) * M * OC // blockIdz.x -> split_k dim
103
+ + (((int)blockIdx_y) % j_factors1) * N + ((int)threadIdx.y) * (N / 2) +
104
+ (((int)threadIdx.x) % 4) * 2;
105
106
// preload s.f. and zeros
107
int k_bound = (IC / 32 + split_k_iters - 1) / split_k_iters;
@@ -115,57 +110,83 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
110
int k_0_0 = _k_0_0 * split_k_iters + blockIdx_z;
111
__syncthreads();
112
// TODO: Haotian: blockIdx_y / j_factors1 in A loading to support bsz > 16
118
- if (ld_A_flag)
119
- {
113
+ if (ld_A_flag) {
114
*(uint4*)(A_shared_ptr) = *(uint4*)(A_ptr + (k_0_0 * 32));
121
- }
122
- else
123
- {
115
+ } else {
116
*(uint4*)(A_shared_ptr) = make_uint4(0, 0, 0, 0);
117
}
118
119
// for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < 2; ++ax0_ax1_fused_0) {
120
uint32_t zeros_loaded = *(uint32_t*)(zeros_ptr + k_0_0 * 32 / G * (OC / 8));
121
uint4 B_loaded_zero = dequantize_s4_to_fp16x2(zeros_loaded);
130
- uint4 B_loaded_scale = *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
122
+ uint4 B_loaded_scale =
123
+ *(uint4*)(scaling_factors_ptr + k_0_0 * 32 / G * (OC));
124
/*
132
- if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 && threadIdx.y == 0){
133
- printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x, B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x, B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
125
+ if (blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 0 &&
126
+ threadIdx.y == 0){ printf("%x %x %x %x %x %x %x %x\n", B_loaded_scale.x,
127
+ B_loaded_scale.y, B_loaded_scale.z, B_loaded_scale.w, B_loaded_zero.x,
128
+ B_loaded_zero.y, B_loaded_zero.z, B_loaded_zero.w);
129
}
130
*/
131
// uint4 B_loaded_scale = make_uint4(0, 0, 0, 0);
132
int* B_ptr_local = B_ptr + k_0_0 * 32 * (OC / 8);
133
134
for (int ax0_ax1_fused_0 = 0; ax0_ax1_fused_0 < N / 16; ++ax0_ax1_fused_0) {
140
-
135
// B: 32 x 136 (128+8) float16
136
// each warp: 32 x 4
143
- // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus zero -> WB UINT4
144
- // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) * 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15) * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 * 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) * 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) * 8)));
145
- // row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
146
- uint32_t B_loaded = *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
137
+ // each thr: read 32 bit -> convert to 8xFP16 (a UINT4) -> scale and minus
138
+ // zero -> WB UINT4
139
+ // *(uint4*)(B_shared + ((((ax0_ax1_fused_0 * 544) + (((int)threadIdx.y) *
140
+ // 272)) + ((((int)threadIdx.x) >> 4) * 136)) + ((((int)threadIdx.x) & 15)
141
+ // * 8))) = *(uint4*)(B + ((((((k_0_0 * 163840) + (ax0_ax1_fused_0 *
142
+ // 20480)) + (((int)threadIdx.y) * 10240)) + ((((int)threadIdx.x) >> 4) *
143
+ // 5120)) + (((int)blockIdx_y) * 128)) + ((((int)threadIdx.x) & 15) *
144
+ // 8))); row stride in shared memory: (NWARPS * 32 * 8 / cta_N)
145
+ uint32_t B_loaded =
146
+ *(uint32_t*)(B_ptr_local + ax0_ax1_fused_0 * row_stride * (OC / 8));
147
uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
148
- //uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N / 8)) * 8);
148
+ // uint4 B_loaded_zero = *(uint4*)(zeros_shared + (threadIdx.x % (cta_N /
149
+ // 8)) * 8);
150
150
- // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x % (cta_N / 8)) * 8);
151
+ // uint4 B_loaded_scale = *(uint4*)(scaling_factors_shared + (threadIdx.x
152
+ // % (cta_N / 8)) * 8);
153
// - zero and * scale
152
- // TODO (Haotian): can save 4 assembly instructions if sormulate as deq = q * scale - zero * scale.
153
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
154
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
155
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
156
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
157
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
158
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
159
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
160
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
154
+ // TODO (Haotian): can save 4 assembly instructions if sormulate as deq =
155
+ // q * scale - zero * scale.
156
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
157
+ : "=r"(B_loaded_fp16.x)
158
+ : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
159
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
160
+ : "=r"(B_loaded_fp16.x)
161
+ : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
162
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
163
+ : "=r"(B_loaded_fp16.y)
164
+ : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
165
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
166
+ : "=r"(B_loaded_fp16.y)
167
+ : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
168
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
169
+ : "=r"(B_loaded_fp16.z)
170
+ : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
171
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
172
+ : "=r"(B_loaded_fp16.z)
173
+ : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
174
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
175
+ : "=r"(B_loaded_fp16.w)
176
+ : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
177
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
178
+ : "=r"(B_loaded_fp16.w)
179
+ : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
180
/*
162
- if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 == 0 && threadIdx.x == 17 && threadIdx.y == 0){
163
- printf("[x] %X %X %X %X\n", B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
181
+ if (ax0_ax1_fused_0 == 0 && blockIdx_z == 0 && blockIdx_y == 0 && k_0_0 ==
182
+ 0 && threadIdx.x == 17 && threadIdx.y == 0){ printf("[x] %X %X %X %X\n",
183
+ B_loaded_fp16.x, B_loaded_fp16.y, B_loaded_fp16.z, B_loaded_fp16.w);
184
}
185
*/
186
187
// write back
168
- *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) = B_loaded_fp16;
188
+ *(uint4*)(B_shared_ptr + ax0_ax1_fused_0 * row_stride * (N + 8)) =
189
+ B_loaded_fp16;
190
}
191
__syncthreads();
192
@@ -173,112 +194,179 @@ __global__ void __launch_bounds__(64) gemm_forward_4bit_cuda_m16nXk32(
194
{
195
unsigned int addr;
196
__asm__ __volatile__(
176
- "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
177
- : "=r"(addr)
178
- : "l"((void *)((&(A_shared[(k_0_1 * 16)])) + (((((int)threadIdx.x) & 15) * 40) + ((((int)threadIdx.x) >> 4) * 8))))
179
- );
180
-
197
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, "
198
+ "addr; }\n"
199
+ : "=r"(addr)
200
+ : "l"((void*)((&(A_shared[(k_0_1 * 16)])) +
201
+ (((((int)threadIdx.x) & 15) * 40) +
202
+ ((((int)threadIdx.x) >> 4) * 8)))));
203
204
__asm__ __volatile__(
183
- "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
184
- "{%0, %1, %2, %3}, [%4];\n"
185
- : "=r"(((unsigned *)(A_shared_warp + 0))[0]), "=r"(((unsigned *)(A_shared_warp + 0))[1]), "=r"(((unsigned *)(A_shared_warp + 0))[2]), "=r"(((unsigned *)(A_shared_warp + 0))[3])
186
- : "r"(addr)
187
- );
205
+ "ldmatrix.sync.aligned.m8n8.x4.shared.b16"
206
+ "{%0, %1, %2, %3}, [%4];\n"
207
+ : "=r"(((unsigned*)(A_shared_warp + 0))[0]),
208
+ "=r"(((unsigned*)(A_shared_warp + 0))[1]),
209
+ "=r"(((unsigned*)(A_shared_warp + 0))[2]),
210
+ "=r"(((unsigned*)(A_shared_warp + 0))[3])
211
+ : "r"(addr));
212
}
213
214
for (int ax1_0 = 0; ax1_0 < N / 32; ++ax1_0) {
215
{
216
unsigned int addr;
217
__asm__ __volatile__(
194
- "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, addr; }\n"
195
- : "=r"(addr)
196
- : "l"((void *)((&(B_shared[(((k_0_1 * (N * 16 + 128)) + (((int)threadIdx.y) * (N / 2))) + (ax1_0 * 16))])) + (((((int)threadIdx.x) & 15) * (N + 8)) + ((((int)threadIdx.x) >> 4) * 8))))
197
- );
218
+ "{ .reg .u64 addr; cvta.to.shared.u64 addr, %1; cvt.u32.u64 %0, "
219
+ "addr; }\n"
220
+ : "=r"(addr)
221
+ : "l"((void*)((&(B_shared[(((k_0_1 * (N * 16 + 128)) +
222
+ (((int)threadIdx.y) * (N / 2))) +
223
+ (ax1_0 * 16))])) +
224
+ (((((int)threadIdx.x) & 15) * (N + 8)) +
225
+ ((((int)threadIdx.x) >> 4) * 8)))));
226
__asm__ __volatile__(
199
- "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
200
- "{%0, %1, %2, %3}, [%4];\n"
201
- : "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[0]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[1]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[2]), "=r"(((unsigned *)(B_shared_warp + (ax1_0 * 8)))[3])
202
- : "r"(addr)
203
- );
227
+ "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16"
228
+ "{%0, %1, %2, %3}, [%4];\n"
229
+ : "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[0]),
230
+ "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[1]),
231
+ "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[2]),
232
+ "=r"(((unsigned*)(B_shared_warp + (ax1_0 * 8)))[3])
233
+ : "r"(addr));
234
}
235
}
236
for (int j_0_4 = 0; j_0_4 < N / 32; ++j_0_4) {
207
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
237
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
238
{
239
__asm__ __volatile__(
210
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
211
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
212
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
213
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
240
+ "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
241
+ "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
242
+ : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
243
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
244
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
245
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
246
+ : "r"(((unsigned*)(A_shared_warp + 0))[0]),
247
+ "r"(((unsigned*)(A_shared_warp + 0))[1]),
248
+ "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[0]),
249
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
250
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
251
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
252
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
253
}
254
255
{
256
__asm__ __volatile__(
218
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
219
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
220
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
221
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
257
+ "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
258
+ "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
259
+ : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
260
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
261
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
262
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
263
+ : "r"(((unsigned*)(A_shared_warp + 0))[0]),
264
+ "r"(((unsigned*)(A_shared_warp + 0))[1]),
265
+ "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]),
266
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
267
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
268
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
269
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
270
}
271
272
{
273
__asm__ __volatile__(
226
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
227
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
228
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
229
- : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
274
+ "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
275
+ "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
276
+ : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
277
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
278
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
279
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
280
+ : "r"(((unsigned*)(A_shared_warp + 0))[2]),
281
+ "r"(((unsigned*)(A_shared_warp + 0))[3]),
282
+ "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[1]),
283
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
284
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
285
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
286
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
287
}
288
289
{
290
__asm__ __volatile__(
234
- "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
235
- "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
236
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
237
- : "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
291
+ "mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32"
292
+ "{%0, %1, %2, %3}, {%4, %5}, {%6}, {%7, %8, %9, %10};\n"
293
+ : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
294
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
295
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
296
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
297
+ : "r"(((unsigned*)(A_shared_warp + 0))[2]),
298
+ "r"(((unsigned*)(A_shared_warp + 0))[3]),
299
+ "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]),
300
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
301
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
302
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
303
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
304
}
239
-#else
305
+ #else
306
{
307
__asm__ __volatile__(
242
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
243
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
244
- : "=f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "=f"(((float *)(C_warp + (j_0_4 * 8)))[3])
245
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[0]), "r"(((unsigned *)(B_shared_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[0]), "f"(((float *)(C_warp + (j_0_4 * 8)))[1]), "f"(((float *)(C_warp + (j_0_4 * 8)))[2]), "f"(((float *)(C_warp + (j_0_4 * 8)))[3]));
308
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
309
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, "
310
+ "%13};\n"
311
+ : "=f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
312
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
313
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
314
+ "=f"(((float*)(C_warp + (j_0_4 * 8)))[3])
315
+ : "r"(((unsigned*)(A_shared_warp + 0))[0]),
316
+ "r"(((unsigned*)(A_shared_warp + 0))[1]),
317
+ "r"(((unsigned*)(A_shared_warp + 0))[2]),
318
+ "r"(((unsigned*)(A_shared_warp + 0))[3]),
319
+ "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[0]),
320
+ "r"(((unsigned*)(B_shared_warp + (j_0_4 * 8)))[1]),
321
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[0]),
322
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[1]),
323
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[2]),
324
+ "f"(((float*)(C_warp + (j_0_4 * 8)))[3]));
325
}
326
327
{
328
__asm__ __volatile__(
250
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
251
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
252
- : "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "=f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3])
253
- : "r"(((unsigned *)(A_shared_warp + 0))[0]), "r"(((unsigned *)(A_shared_warp + 0))[1]), "r"(((unsigned *)(A_shared_warp + 0))[2]), "r"(((unsigned *)(A_shared_warp + 0))[3]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]), "r"(((unsigned *)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[0]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[1]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[2]), "f"(((float *)(C_warp + ((j_0_4 * 8) + 4)))[3]));
329
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32"
330
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, "
331
+ "%13};\n"
332
+ : "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
333
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
334
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
335
+ "=f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3])
336
+ : "r"(((unsigned*)(A_shared_warp + 0))[0]),
337
+ "r"(((unsigned*)(A_shared_warp + 0))[1]),
338
+ "r"(((unsigned*)(A_shared_warp + 0))[2]),
339
+ "r"(((unsigned*)(A_shared_warp + 0))[3]),
340
+ "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[0]),
341
+ "r"(((unsigned*)(B_shared_warp + ((j_0_4 * 8) + 4)))[1]),
342
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[0]),
343
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[1]),
344
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[2]),
345
+ "f"(((float*)(C_warp + ((j_0_4 * 8) + 4)))[3]));
346
}
347
256
-#endif
348
+ #endif
349
}
350
}
351
}
352
261
-// TODO: Shang: Hoist loop invariance.
353
+ // TODO: Shang: Hoist loop invariance.
354
for (int ax1_0_1 = 0; ax1_0_1 < 4; ++ax1_0_1) {
355
for (int local_id = 0; local_id < 8; ++local_id) {
264
- int row_offset = (((int)blockIdx_y) / j_factors1) * 16 + ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
265
- if (row_offset < M)
266
- {
267
- *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 + local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
356
+ int row_offset = (((int)blockIdx_y) / j_factors1) * 16 +
357
+ ((int)threadIdx.x) / 4 + (local_id % 4) / 2 * 8;
358
+ if (row_offset < M) {
359
+ *(C_ptr + ax1_0_1 * 16 + row_offset * OC + (local_id / 4) * 8 +
360
+ local_id % 2) = __float2half(C_warp[(ax1_0_1 * 8) + local_id]);
361
}
362
}
363
}
364
#endif
365
}
366
274
-__global__ void __launch_bounds__(64) dequantize_weights(
275
- int* __restrict__ B,
276
- half* __restrict__ scaling_factors,
277
- int* __restrict__ zeros,
278
- half* __restrict__ C,
279
- int G
280
-)
281
-{
367
+__global__ void __launch_bounds__(64)
368
+ dequantize_weights(int* __restrict__ B, half* __restrict__ scaling_factors,
369
+ int* __restrict__ zeros, half* __restrict__ C, int G) {
370
int j_factors1 = 4;
371
int row_stride2 = 4;
372
int split_k_iters = 1;
@@ -310,14 +398,30 @@ __global__ void __launch_bounds__(64) dequantize_weights(
398
399
uint32_t B_loaded = *(uint32_t*)B_ptr2;
400
uint4 B_loaded_fp16 = dequantize_s4_to_fp16x2(B_loaded);
313
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
314
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.x) : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
315
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
316
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.y) : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
317
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
318
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.z) : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
319
- asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
320
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(B_loaded_fp16.w) : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
401
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
402
+ : "=r"(B_loaded_fp16.x)
403
+ : "r"(B_loaded_fp16.x), "r"(B_loaded_zero.x));
404
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
405
+ : "=r"(B_loaded_fp16.x)
406
+ : "r"(B_loaded_fp16.x), "r"(B_loaded_scale.x), "r"(ZERO));
407
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
408
+ : "=r"(B_loaded_fp16.y)
409
+ : "r"(B_loaded_fp16.y), "r"(B_loaded_zero.y));
410
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
411
+ : "=r"(B_loaded_fp16.y)
412
+ : "r"(B_loaded_fp16.y), "r"(B_loaded_scale.y), "r"(ZERO));
413
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
414
+ : "=r"(B_loaded_fp16.z)
415
+ : "r"(B_loaded_fp16.z), "r"(B_loaded_zero.z));
416
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
417
+ : "=r"(B_loaded_fp16.z)
418
+ : "r"(B_loaded_fp16.z), "r"(B_loaded_scale.z), "r"(ZERO));
419
+ asm volatile("sub.f16x2 %0, %1, %2;\n"
420
+ : "=r"(B_loaded_fp16.w)
421
+ : "r"(B_loaded_fp16.w), "r"(B_loaded_zero.w));
422
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n"
423
+ : "=r"(B_loaded_fp16.w)
424
+ : "r"(B_loaded_fp16.w), "r"(B_loaded_scale.w), "r"(ZERO));
425
426
*(uint4*)B_shared_ptr2 = B_loaded_fp16;
427
@@ -326,58 +430,57 @@ __global__ void __launch_bounds__(64) dequantize_weights(
430
}
431
}
432
329
-} // namespace awq
330
-} // namespace vllm
331
-
332
-torch::Tensor awq_dequantize(
333
- torch::Tensor _kernel,
334
- torch::Tensor _scaling_factors,
335
- torch::Tensor _zeros,
336
- int split_k_iters,
337
- int thx,
338
- int thy)
339
-{
340
- int in_c = _kernel.size(0);
341
- int qout_c = _kernel.size(1);
342
- int out_c = qout_c * 8;
343
- int G = in_c / _scaling_factors.size(0);
344
-
345
- int x_thread = thx;
346
- int y_thread = thy;
347
-
348
- int x_blocks = 1;
349
- int y_blocks = 1;
350
- if (thx==0) {
351
- x_thread = qout_c;
352
- }
353
- if (thy==0) {
354
- y_thread = in_c;
355
- }
356
- if (thx==0 && thy==0) {
357
- x_thread = 8;
358
- y_thread = 8;
359
- x_blocks = (int)(qout_c / 8);
360
- y_blocks = (int)(in_c / 8);
361
- }
433
+} // namespace awq
434
+} // namespace vllm
435
+
436
+torch::Tensor awq_dequantize(torch::Tensor _kernel,
437
+ torch::Tensor _scaling_factors,
438
+ torch::Tensor _zeros, int split_k_iters, int thx,
439
+ int thy) {
440
+ int in_c = _kernel.size(0);
441
+ int qout_c = _kernel.size(1);
442
+ int out_c = qout_c * 8;
443
+ int G = in_c / _scaling_factors.size(0);
444
+
445
+ int x_thread = thx;
446
+ int y_thread = thy;
447
+
448
+ int x_blocks = 1;
449
+ int y_blocks = 1;
450
+ if (thx == 0) {
451
+ x_thread = qout_c;
452
+ }
453
+ if (thy == 0) {
454
+ y_thread = in_c;
455
+ }
456
+ if (thx == 0 && thy == 0) {
457
+ x_thread = 8;
458
+ y_thread = 8;
459
+ x_blocks = (int)(qout_c / 8);
460
+ y_blocks = (int)(in_c / 8);
461
+ }
462
363
- const at::cuda::OptionalCUDAGuard device_guard(device_of(_scaling_factors));
463
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(_scaling_factors));
464
365
- auto options = torch::TensorOptions().dtype(_scaling_factors.dtype()).device(_scaling_factors.device());
366
- at::Tensor _de_kernel = torch::empty({in_c, out_c}, options);
465
+ auto options = torch::TensorOptions()
466
+ .dtype(_scaling_factors.dtype())
467
+ .device(_scaling_factors.device());
468
+ at::Tensor _de_kernel = torch::empty({in_c, out_c}, options);
469
368
- auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
369
- auto de_kernel = reinterpret_cast<half*>(_de_kernel.data_ptr<at::Half>());
370
- auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
371
- auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
470
+ auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
471
+ auto de_kernel = reinterpret_cast<half*>(_de_kernel.data_ptr<at::Half>());
472
+ auto scaling_factors =
473
+ reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
474
+ auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
475
373
- dim3 num_blocks(x_blocks, y_blocks);
374
- dim3 threads_per_block(x_thread, y_thread);
476
+ dim3 num_blocks(x_blocks, y_blocks);
477
+ dim3 threads_per_block(x_thread, y_thread);
478
376
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
377
- vllm::awq::dequantize_weights<<<num_blocks, threads_per_block, 0, stream>>>(
378
- kernel, scaling_factors, zeros, de_kernel, G);
479
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
480
+ vllm::awq::dequantize_weights<<<num_blocks, threads_per_block, 0, stream>>>(
481
+ kernel, scaling_factors, zeros, de_kernel, G);
482
380
- return _de_kernel;
483
+ return _de_kernel;
484
}
485
486
// in_feats: M, IC [float16]
@@ -386,61 +489,61 @@ torch::Tensor awq_dequantize(
489
// zeros: IC // G, OC // 8 [int32] -> cast to IC // G, OC [uint4b]
490
// assume that batch_size < 16 for now
491
389
-torch::Tensor awq_gemm(
390
- torch::Tensor _in_feats,
391
- torch::Tensor _kernel,
392
- torch::Tensor _scaling_factors,
393
- torch::Tensor _zeros,
394
- int split_k_iters)
395
-{
396
- int num_in_feats = _in_feats.size(0);
397
- int num_in_channels = _in_feats.size(1);
398
- const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
399
-
400
- auto options = torch::TensorOptions().dtype(_in_feats.dtype()).device(_in_feats.device());
401
- at::Tensor _out_feats = torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
402
- int num_out_feats = _out_feats.size(-2);
403
- int num_out_channels = _out_feats.size(-1);
404
-
405
- auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
406
- auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
407
- auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
408
- auto scaling_factors = reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
409
- auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
410
- int group_size = num_in_channels / _scaling_factors.size(0);
411
-
412
- if (num_out_channels % 64 != 0)
413
- throw std::invalid_argument("OC is not multiple of cta_N = 64");
414
- if (num_out_channels % 8 != 0)
415
- throw std::invalid_argument("OC is not multiple of pack_num = 8");
416
- if (group_size % 32 != 0)
417
- throw std::invalid_argument("Group size should be a multiple of 32");
418
- if (num_out_channels % group_size != 0)
419
- throw std::invalid_argument("OC is not multiple of Group size");
420
-
421
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
422
- if (num_out_channels % 128 == 0)
423
- {
424
- int j_factors1 = num_out_channels / 128 / 1;
425
- dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
426
- // threadIdx.x: 32
427
- // threadIdx.y: i_factors[2] * j_factors[2]
428
- dim3 threads_per_block(32, 2);
429
- vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128><<<num_blocks, threads_per_block, 0, stream>>>(
430
- group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
431
- num_out_channels, out_feats);
432
- }
433
- else if (num_out_channels % 64 == 0)
434
- {
435
- int j_factors1 = num_out_channels / 64 / 1;
436
- dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
437
-
438
- // threadIdx.x: 32
439
- // threadIdx.y: i_factors[2] * j_factors[2]
440
- dim3 threads_per_block(32, 2);
441
- vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64><<<num_blocks, threads_per_block, 0, stream>>>(
442
- group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros, num_in_feats, num_in_channels,
443
- num_out_channels, out_feats);
444
- }
445
- return _out_feats.sum(0);
492
+torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
493
+ torch::Tensor _scaling_factors, torch::Tensor _zeros,
494
+ int split_k_iters) {
495
+ int num_in_feats = _in_feats.size(0);
496
+ int num_in_channels = _in_feats.size(1);
497
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(_in_feats));
498
+
499
+ auto options = torch::TensorOptions()
500
+ .dtype(_in_feats.dtype())
501
+ .device(_in_feats.device());
502
+ at::Tensor _out_feats =
503
+ torch::empty({split_k_iters, num_in_feats, _kernel.size(1) * 8}, options);
504
+ int num_out_feats = _out_feats.size(-2);
505
+ int num_out_channels = _out_feats.size(-1);
506
+
507
+ auto in_feats = reinterpret_cast<half*>(_in_feats.data_ptr<at::Half>());
508
+ auto kernel = reinterpret_cast<int*>(_kernel.data_ptr<int>());
509
+ auto out_feats = reinterpret_cast<half*>(_out_feats.data_ptr<at::Half>());
510
+ auto scaling_factors =
511
+ reinterpret_cast<half*>(_scaling_factors.data_ptr<at::Half>());
512
+ auto zeros = reinterpret_cast<int*>(_zeros.data_ptr<int>());
513
+ int group_size = num_in_channels / _scaling_factors.size(0);
514
+
515
+ if (num_out_channels % 64 != 0)
516
+ throw std::invalid_argument("OC is not multiple of cta_N = 64");
517
+ if (num_out_channels % 8 != 0)
518
+ throw std::invalid_argument("OC is not multiple of pack_num = 8");
519
+ if (group_size % 32 != 0)
520
+ throw std::invalid_argument("Group size should be a multiple of 32");
521
+ if (num_out_channels % group_size != 0)
522
+ throw std::invalid_argument("OC is not multiple of Group size");
523
+
524
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
525
+ if (num_out_channels % 128 == 0) {
526
+ int j_factors1 = num_out_channels / 128 / 1;
527
+ dim3 num_blocks((num_out_feats + 16 - 1) / 16 * j_factors1 * split_k_iters);
528
+ // threadIdx.x: 32
529
+ // threadIdx.y: i_factors[2] * j_factors[2]
530
+ dim3 threads_per_block(32, 2);
531
+ vllm::awq::gemm_forward_4bit_cuda_m16nXk32<128>
532
+ <<<num_blocks, threads_per_block, 0, stream>>>(
533
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros,
534
+ num_in_feats, num_in_channels, num_out_channels, out_feats);
535
+ } else if (num_out_channels % 64 == 0) {
536
+ int j_factors1 = num_out_channels / 64 / 1;
537
+ dim3 num_blocks(1 * (num_out_feats + 16 - 1) / 16 * j_factors1 *
538
+ split_k_iters);
539
+
540
+ // threadIdx.x: 32
541
+ // threadIdx.y: i_factors[2] * j_factors[2]
542
+ dim3 threads_per_block(32, 2);
543
+ vllm::awq::gemm_forward_4bit_cuda_m16nXk32<64>
544
+ <<<num_blocks, threads_per_block, 0, stream>>>(
545
+ group_size, split_k_iters, in_feats, kernel, scaling_factors, zeros,
546
+ num_in_feats, num_in_channels, num_out_channels, out_feats);
547
+ }
548
+ return _out_feats.sum(0);
549
}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+19
-19
@@ -117,10 +117,10 @@ struct cutlass_2x_gemm {
117
};
118
119
template <typename Gemm>
120
-void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
121
- torch::Tensor const &b,
122
- torch::Tensor const &a_scales,
123
- torch::Tensor const &b_scales) {
120
+void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
121
+ torch::Tensor const& b,
122
+ torch::Tensor const& a_scales,
123
+ torch::Tensor const& b_scales) {
124
using ElementAB = typename Gemm::ElementAB;
125
using ElementD = typename Gemm::ElementD;
126
@@ -136,9 +136,9 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
136
using StrideC = Stride<int64_t, Int<1>, Int<0>>;
137
StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
138
139
- auto a_ptr = static_cast<ElementAB const *>(a.data_ptr());
140
- auto b_ptr = static_cast<ElementAB const *>(b.data_ptr());
141
- auto c_ptr = static_cast<ElementD *>(out.data_ptr());
139
+ auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
140
+ auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
141
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
142
143
auto a_scales_ptr = a_scales.data_ptr<float>();
144
auto b_scales_ptr = b_scales.data_ptr<float>();
@@ -196,10 +196,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
196
197
} // namespace
198
199
-void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
200
- torch::Tensor const &b,
201
- torch::Tensor const &a_scales,
202
- torch::Tensor const &b_scales) {
199
+void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
200
+ torch::Tensor const& b,
201
+ torch::Tensor const& a_scales,
202
+ torch::Tensor const& b_scales) {
203
TORCH_CHECK(a.dtype() == torch::kInt8);
204
TORCH_CHECK(b.dtype() == torch::kInt8);
205
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -223,10 +223,10 @@ void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
223
}
224
}
225
226
-void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
227
- torch::Tensor const &b,
228
- torch::Tensor const &a_scales,
229
- torch::Tensor const &b_scales) {
226
+void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
227
+ torch::Tensor const& b,
228
+ torch::Tensor const& a_scales,
229
+ torch::Tensor const& b_scales) {
230
TORCH_CHECK(a.dtype() == torch::kInt8);
231
TORCH_CHECK(b.dtype() == torch::kInt8);
232
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -250,10 +250,10 @@ void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
250
}
251
}
252
253
-void cutlass_scaled_mm_dq_sm89(torch::Tensor &out, torch::Tensor const &a,
254
- torch::Tensor const &b,
255
- torch::Tensor const &a_scales,
256
- torch::Tensor const &b_scales) {
253
+void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
254
+ torch::Tensor const& b,
255
+ torch::Tensor const& a_scales,
256
+ torch::Tensor const& b_scales) {
257
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
258
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
259
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+11
-11
@@ -120,10 +120,10 @@ struct cutlass_3x_gemm {
120
};
121
122
template <typename Gemm>
123
-void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
124
- torch::Tensor const &b,
125
- torch::Tensor const &a_scales,
126
- torch::Tensor const &b_scales) {
123
+void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
124
+ torch::Tensor const& b,
125
+ torch::Tensor const& a_scales,
126
+ torch::Tensor const& b_scales) {
127
using ElementAB = typename Gemm::ElementAB;
128
using ElementD = typename Gemm::ElementD;
129
@@ -146,12 +146,12 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
146
using GemmKernel = typename Gemm::GemmKernel;
147
typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
148
149
- auto a_ptr = static_cast<ElementAB *>(a.data_ptr());
150
- auto b_ptr = static_cast<ElementAB *>(b.data_ptr());
149
+ auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
150
+ auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
151
typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
152
b_stride};
153
154
- auto c_ptr = static_cast<ElementD *>(out.data_ptr());
154
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
155
typename GemmKernel::EpilogueArguments epilogue_args{
156
{}, c_ptr, c_stride, c_ptr, c_stride};
157
@@ -183,10 +183,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
183
}
184
} // namespace
185
186
-void cutlass_scaled_mm_dq_sm90(torch::Tensor &out, torch::Tensor const &a,
187
- torch::Tensor const &b,
188
- torch::Tensor const &a_scales,
189
- torch::Tensor const &b_scales) {
186
+void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
187
+ torch::Tensor const& b,
188
+ torch::Tensor const& a_scales,
189
+ torch::Tensor const& b_scales) {
190
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
191
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
192
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+24
-23
@@ -2,29 +2,29 @@
2
#include <cuda_runtime.h>
3
#include <torch/extension.h>
4
5
-void cutlass_scaled_mm_dq_sm75(torch::Tensor &c, torch::Tensor const &a,
6
- torch::Tensor const &b,
7
- torch::Tensor const &a_scales,
8
- torch::Tensor const &b_scales);
5
+void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
6
+ torch::Tensor const& b,
7
+ torch::Tensor const& a_scales,
8
+ torch::Tensor const& b_scales);
9
10
-void cutlass_scaled_mm_dq_sm80(torch::Tensor &c, torch::Tensor const &a,
11
- torch::Tensor const &b,
12
- torch::Tensor const &a_scales,
13
- torch::Tensor const &b_scales);
10
+void cutlass_scaled_mm_dq_sm80(torch::Tensor& c, torch::Tensor const& a,
11
+ torch::Tensor const& b,
12
+ torch::Tensor const& a_scales,
13
+ torch::Tensor const& b_scales);
14
15
-void cutlass_scaled_mm_dq_sm89(torch::Tensor &c, torch::Tensor const &a,
16
- torch::Tensor const &b,
17
- torch::Tensor const &a_scales,
18
- torch::Tensor const &b_scales);
15
+void cutlass_scaled_mm_dq_sm89(torch::Tensor& c, torch::Tensor const& a,
16
+ torch::Tensor const& b,
17
+ torch::Tensor const& a_scales,
18
+ torch::Tensor const& b_scales);
19
20
-void cutlass_scaled_mm_dq_sm90(torch::Tensor &c, torch::Tensor const &a,
21
- torch::Tensor const &b,
22
- torch::Tensor const &a_scales,
23
- torch::Tensor const &b_scales);
20
+void cutlass_scaled_mm_dq_sm90(torch::Tensor& c, torch::Tensor const& a,
21
+ torch::Tensor const& b,
22
+ torch::Tensor const& a_scales,
23
+ torch::Tensor const& b_scales);
24
25
-void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
26
- torch::Tensor const &b, torch::Tensor const &a_scales,
27
- torch::Tensor const &b_scales) {
25
+void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
26
+ torch::Tensor const& b, torch::Tensor const& a_scales,
27
+ torch::Tensor const& b_scales) {
28
int32_t major_capability;
29
int32_t minor_capability;
30
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
@@ -36,14 +36,15 @@ void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
36
// Checks for conformality
37
TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
38
TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
39
- b.size(1) == c.size(1));
39
+ b.size(1) == c.size(1));
40
TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
41
TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
42
43
// Check for strides and alignment
44
- TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45
- TORCH_CHECK(b.stride(0) == 1); // Column-major
46
- TORCH_CHECK(c.stride(0) % 16 == 0 && b.stride(1) % 16 == 0); // 16 Byte Alignment
44
+ TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45
+ TORCH_CHECK(b.stride(0) == 1); // Column-major
46
+ TORCH_CHECK(c.stride(0) % 16 == 0 &&
47
+ b.stride(1) % 16 == 0); // 16 Byte Alignment
48
TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
49
50
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
csrc/quantization/fp8/amd/hip_float8.h
+93
-123
@@ -1,167 +1,137 @@
1
#pragma once
2
3
#ifdef __HIPCC__
4
-#include <hip/hip_runtime.h>
4
+ #include <hip/hip_runtime.h>
5
#else
6
-#include <type_traits>
7
-#include <stdint.h>
8
-#include <math.h>
9
-#include <iostream>
6
+ #include <type_traits>
7
+ #include <stdint.h>
8
+ #include <math.h>
9
+ #include <iostream>
10
#endif
11
12
#include "hip_float8_impl.h"
13
14
-struct alignas(1) hip_fp8
15
-{
16
- struct from_bits_t
17
- {
18
- };
19
- HIP_FP8_HOST_DEVICE static constexpr from_bits_t from_bits() { return from_bits_t(); }
20
- uint8_t data;
21
-
22
- hip_fp8() = default;
23
- HIP_FP8_HOST_DEVICE constexpr hip_fp8(const hip_fp8&) = default;
24
- HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v) = delete;
25
- explicit HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v, from_bits_t)
26
- : data(v)
27
- {
28
- }
14
+struct alignas(1) hip_fp8 {
15
+ struct from_bits_t {};
16
+ HIP_FP8_HOST_DEVICE static constexpr from_bits_t from_bits() {
17
+ return from_bits_t();
18
+ }
19
+ uint8_t data;
20
+
21
+ hip_fp8() = default;
22
+ HIP_FP8_HOST_DEVICE constexpr hip_fp8(const hip_fp8&) = default;
23
+ HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v) = delete;
24
+ explicit HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v, from_bits_t)
25
+ : data(v) {}
26
27
#ifdef __HIP__MI300__
31
- // NOTE: ON-DEVICE... always optimal bias
32
- explicit HIP_FP8_DEVICE hip_fp8(float v)
33
- : data(hip_fp8_impl::to_fp8_from_fp32(v))
34
- {
35
- }
36
-
37
- explicit HIP_FP8_DEVICE hip_fp8(_Float16 v)
38
- : hip_fp8(static_cast<float>(v))
39
- {
40
- }
41
-
42
- // Host only implementation using s/w simulation
43
- explicit HIP_FP8_HOST
44
-#else // __HIP__MI300__
45
- // both Host and DEVICE for non-MI300 using s/w simulation
46
- explicit HIP_FP8_HOST_DEVICE
47
-#endif // __HIP__MI300__
48
- hip_fp8(float v)
49
- {
50
- data = hip_fp8_impl::to_float8<4, 3, float, true /*negative_zero_nan*/, true /*clip*/>(v);
51
- }
52
-
53
- explicit HIP_FP8_HOST_DEVICE hip_fp8(double v)
54
- : hip_fp8(static_cast<float>(v))
55
- {
56
- }
28
+ // NOTE: ON-DEVICE... always optimal bias
29
+ explicit HIP_FP8_DEVICE hip_fp8(float v)
30
+ : data(hip_fp8_impl::to_fp8_from_fp32(v)) {}
31
+
32
+ explicit HIP_FP8_DEVICE hip_fp8(_Float16 v)
33
+ : hip_fp8(static_cast<float>(v)) {}
34
+
35
+ // Host only implementation using s/w simulation
36
+ explicit HIP_FP8_HOST
37
+#else // __HIP__MI300__
38
+ // both Host and DEVICE for non-MI300 using s/w simulation
39
+ explicit HIP_FP8_HOST_DEVICE
40
+#endif // __HIP__MI300__
41
+ hip_fp8(float v) {
42
+ data = hip_fp8_impl::to_float8<4, 3, float, true /*negative_zero_nan*/,
43
+ true /*clip*/>(v);
44
+ }
45
+
46
+ explicit HIP_FP8_HOST_DEVICE hip_fp8(double v)
47
+ : hip_fp8(static_cast<float>(v)) {}
48
49
#ifdef __HIP__MI300__
59
- // upcast using device specific intrinsic
60
- explicit inline HIP_FP8_DEVICE operator float() const
61
- {
62
- float fval;
63
- uint32_t i32val = static_cast<uint32_t>(data);
64
-
65
- // upcast
66
- asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val));
67
-
68
- return fval;
69
- }
70
-
71
- explicit inline HIP_FP8_HOST operator float() const
72
-#else // __HIP__MI300__
73
- explicit inline HIP_FP8_HOST_DEVICE operator float() const
74
-#endif // __HIP__MI300__
75
- {
76
- return hip_fp8_impl::from_float8<4, 3, float, true /*negative_zero_nan*/>(data);
77
- }
50
+ // upcast using device specific intrinsic
51
+ explicit inline HIP_FP8_DEVICE operator float() const {
52
+ float fval;
53
+ uint32_t i32val = static_cast<uint32_t>(data);
54
+
55
+ // upcast
56
+ asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0"
57
+ : "=v"(fval)
58
+ : "v"(i32val));
59
+
60
+ return fval;
61
+ }
62
+
63
+ explicit inline HIP_FP8_HOST operator float() const
64
+#else // __HIP__MI300__
65
+ explicit inline HIP_FP8_HOST_DEVICE operator float() const
66
+#endif // __HIP__MI300__
67
+ {
68
+ return hip_fp8_impl::from_float8<4, 3, float, true /*negative_zero_nan*/>(
69
+ data);
70
+ }
71
};
72
80
-namespace std
81
-{
82
-inline hip_fp8 sin(hip_fp8 a)
83
-{
84
- return hip_fp8(sinf(float(a)));
85
-}
86
-inline hip_fp8 cos(hip_fp8 a)
87
-{
88
- return hip_fp8(cosf(float(a)));
89
-}
90
-HIP_FP8_HOST_DEVICE constexpr hip_fp8 real(const hip_fp8& a)
91
-{
92
- return a;
93
-}
94
-} // namespace std
73
+namespace std {
74
+inline hip_fp8 sin(hip_fp8 a) { return hip_fp8(sinf(float(a))); }
75
+inline hip_fp8 cos(hip_fp8 a) { return hip_fp8(cosf(float(a))); }
76
+HIP_FP8_HOST_DEVICE constexpr hip_fp8 real(const hip_fp8& a) { return a; }
77
+} // namespace std
78
79
// Special operator overloading
97
-inline std::ostream& operator<<(std::ostream& os, const hip_fp8& f8)
98
-{
99
- return os << float(f8);
80
+inline std::ostream& operator<<(std::ostream& os, const hip_fp8& f8) {
81
+ return os << float(f8);
82
}
83
84
// all + operator overloading with mixed types
103
-// mixed types, always converts to f32, does computation in f32, and returns float
104
-inline HIP_FP8_HOST_DEVICE float operator+(const float fa, hip_fp8 b)
105
-{
106
- return (fa + float(b));
85
+// mixed types, always converts to f32, does computation in f32, and returns
86
+// float
87
+inline HIP_FP8_HOST_DEVICE float operator+(const float fa, hip_fp8 b) {
88
+ return (fa + float(b));
89
}
90
109
-inline HIP_FP8_HOST_DEVICE float operator+(hip_fp8 a, const float fb)
110
-{
111
- return (float(a) + fb);
91
+inline HIP_FP8_HOST_DEVICE float operator+(hip_fp8 a, const float fb) {
92
+ return (float(a) + fb);
93
}
94
114
-inline HIP_FP8_HOST_DEVICE hip_fp8 operator+(hip_fp8 a, hip_fp8 b)
115
-{
116
- return hip_fp8(float(a) + float(b));
95
+inline HIP_FP8_HOST_DEVICE hip_fp8 operator+(hip_fp8 a, hip_fp8 b) {
96
+ return hip_fp8(float(a) + float(b));
97
}
98
119
-inline HIP_FP8_HOST_DEVICE hip_fp8& operator+=(hip_fp8& a, hip_fp8 b)
120
-{
121
- return a = hip_fp8(float(a) + float(b));
99
+inline HIP_FP8_HOST_DEVICE hip_fp8& operator+=(hip_fp8& a, hip_fp8 b) {
100
+ return a = hip_fp8(float(a) + float(b));
101
}
102
103
// overloading multiplication, always returns float,
125
-inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, hip_fp8 b)
126
-{
127
- return float(a) * float(b);
104
+inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, hip_fp8 b) {
105
+ return float(a) * float(b);
106
}
107
130
-inline HIP_FP8_HOST_DEVICE float operator*(float a, hip_fp8 b)
131
-{
132
- return (a * float(b));
108
+inline HIP_FP8_HOST_DEVICE float operator*(float a, hip_fp8 b) {
109
+ return (a * float(b));
110
}
111
135
-inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, float b)
136
-{
137
- return (float(a) * b);
112
+inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, float b) {
113
+ return (float(a) * b);
114
}
115
140
-inline HIP_FP8_HOST_DEVICE float operator*(int32_t a, hip_fp8 b)
141
-{
142
- return ((float)a * float(b));
116
+inline HIP_FP8_HOST_DEVICE float operator*(int32_t a, hip_fp8 b) {
117
+ return ((float)a * float(b));
118
}
119
145
-inline HIP_FP8_HOST_DEVICE float operator*(double a, hip_fp8 b)
146
-{
147
- return ((float)a * float(b));
120
+inline HIP_FP8_HOST_DEVICE float operator*(double a, hip_fp8 b) {
121
+ return ((float)a * float(b));
122
}
123
124
// overloading for compare
151
-inline HIP_FP8_HOST_DEVICE bool operator==(hip_fp8 a, hip_fp8 b)
152
-{
153
- return (a.data == b.data);
125
+inline HIP_FP8_HOST_DEVICE bool operator==(hip_fp8 a, hip_fp8 b) {
126
+ return (a.data == b.data);
127
}
155
-inline HIP_FP8_HOST_DEVICE bool operator!=(hip_fp8 a, hip_fp8 b)
156
-{
157
- return (a.data != b.data);
128
+inline HIP_FP8_HOST_DEVICE bool operator!=(hip_fp8 a, hip_fp8 b) {
129
+ return (a.data != b.data);
130
}
131
160
-inline HIP_FP8_HOST_DEVICE bool operator>=(hip_fp8 a, hip_fp8 b)
161
-{
162
- return static_cast<float>(a) >= static_cast<float>(b);
132
+inline HIP_FP8_HOST_DEVICE bool operator>=(hip_fp8 a, hip_fp8 b) {
133
+ return static_cast<float>(a) >= static_cast<float>(b);
134
}
164
-inline HIP_FP8_HOST_DEVICE bool operator>(hip_fp8 a, hip_fp8 b)
165
-{
166
- return static_cast<float>(a) > static_cast<float>(b);
135
+inline HIP_FP8_HOST_DEVICE bool operator>(hip_fp8 a, hip_fp8 b) {
136
+ return static_cast<float>(a) > static_cast<float>(b);
137
}
csrc/quantization/fp8/amd/hip_float8_impl.h
+260
-260
@@ -1,316 +1,316 @@
1
#pragma once
2
3
-#if defined(__HIPCC__) && (defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__))
4
-#define __HIP__MI300__
3
+#if defined(__HIPCC__) && \
4
+ (defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__))
5
+ #define __HIP__MI300__
6
#endif
7
8
#ifdef __HIPCC__
8
-#define HIP_FP8_HOST_DEVICE __host__ __device__
9
-#define HIP_FP8_HOST __host__
10
-#define HIP_FP8_DEVICE __device__
9
+ #define HIP_FP8_HOST_DEVICE __host__ __device__
10
+ #define HIP_FP8_HOST __host__
11
+ #define HIP_FP8_DEVICE __device__
12
#else
12
-#define HIP_FP8_HOST_DEVICE
13
-#define HIP_FP8_HOST
14
-#define HIP_FP8_DEVICE
13
+ #define HIP_FP8_HOST_DEVICE
14
+ #define HIP_FP8_HOST
15
+ #define HIP_FP8_DEVICE
16
#endif
17
17
-namespace hip_fp8_impl
18
-{
18
+namespace hip_fp8_impl {
19
20
#ifdef __HIP__MI300__
21
-HIP_FP8_DEVICE uint8_t to_fp8_from_fp32(float v)
22
-{
23
- uint8_t i8data;
24
- union {
25
- float fval;
26
- uint32_t i32val;
27
- uint8_t i8val[4]; // NOTE: not endian independent
28
- } val;
29
-
30
- uint32_t ival = 0;
31
- val.fval = v;
32
-
33
- if ((val.i32val & 0x7F800000) != 0x7F800000) { /// propagate NAN/INF, no clipping
34
- val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0);
35
- }
36
-
37
- ival = __builtin_amdgcn_cvt_pk_fp8_f32(val.fval, val.fval, ival,
38
- false); // false -> WORD0
39
- val.i32val = ival;
40
- i8data = val.i8val[0];
41
-
42
- return i8data;
21
+HIP_FP8_DEVICE uint8_t to_fp8_from_fp32(float v) {
22
+ uint8_t i8data;
23
+ union {
24
+ float fval;
25
+ uint32_t i32val;
26
+ uint8_t i8val[4]; // NOTE: not endian independent
27
+ } val;
28
+
29
+ uint32_t ival = 0;
30
+ val.fval = v;
31
+
32
+ if ((val.i32val & 0x7F800000) !=
33
+ 0x7F800000) { /// propagate NAN/INF, no clipping
34
+ val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0);
35
+ }
36
+
37
+ ival = __builtin_amdgcn_cvt_pk_fp8_f32(val.fval, val.fval, ival,
38
+ false); // false -> WORD0
39
+ val.i32val = ival;
40
+ i8data = val.i8val[0];
41
+
42
+ return i8data;
43
}
44
-#endif // __HIP__MI300__
44
+#endif // __HIP__MI300__
45
46
-HIP_FP8_HOST inline int clz(uint32_t x)
47
-{
48
- return __builtin_clz(x);
49
-}
46
+HIP_FP8_HOST inline int clz(uint32_t x) { return __builtin_clz(x); }
47
#if defined(__HIPCC__) || defined(__CUDA_ARCH__)
51
-HIP_FP8_DEVICE inline int clz(uint32_t x)
52
-{
53
- return __clz(x);
54
-}
48
+HIP_FP8_DEVICE inline int clz(uint32_t x) { return __clz(x); }
49
#endif
50
51
template <int we, int wm, typename T, bool negative_zero_nan, bool clip>
58
-HIP_FP8_HOST_DEVICE uint8_t to_float8(T _x, bool stoch = false, uint32_t rng = 0)
59
-{
52
+HIP_FP8_HOST_DEVICE uint8_t to_float8(T _x, bool stoch = false,
53
+ uint32_t rng = 0) {
54
#ifdef __HIPCC__
61
- constexpr bool is_half = std::is_same<T, _Float16>::value;
55
+ constexpr bool is_half = std::is_same<T, _Float16>::value;
56
#else
63
- constexpr bool is_half = false;
57
+ constexpr bool is_half = false;
58
#endif
65
- constexpr bool is_float = std::is_same<T, float>::value;
66
- static_assert(wm + we == 7, "wm+we==7");
67
- static_assert(is_half || is_float, "Only half and float can be cast to f8");
68
-
69
- const int mfmt = (sizeof(T) == 4) ? 23 : 10;
70
- uint32_t x;
59
+ constexpr bool is_float = std::is_same<T, float>::value;
60
+ static_assert(wm + we == 7, "wm+we==7");
61
+ static_assert(is_half || is_float, "Only half and float can be cast to f8");
62
+
63
+ const int mfmt = (sizeof(T) == 4) ? 23 : 10;
64
+ uint32_t x;
65
+ if (sizeof(T) == 4) {
66
+ x = reinterpret_cast<uint32_t&>(_x);
67
+ } else {
68
+ x = reinterpret_cast<uint16_t&>(_x);
69
+ }
70
+
71
+ uint32_t head, mantissa;
72
+ int exponent, bias;
73
+ uint32_t sign;
74
+
75
+ if (sizeof(T) == 4) {
76
+ head = x & 0xFF800000;
77
+ mantissa = x & 0x7FFFFF;
78
+ exponent = (head >> 23) & 0xFF;
79
+ sign = head >> 31;
80
+ bias = 127;
81
+ } else {
82
+ head = x & 0xFC00;
83
+ mantissa = x & 0x3FF;
84
+ exponent = (head >> 10) & 0x1F;
85
+ sign = head >> 15;
86
+ bias = 15;
87
+ }
88
+
89
+ uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm);
90
+
91
+ // Deal with inf and NaNs
92
+ if (negative_zero_nan) {
93
if (sizeof(T) == 4) {
72
- x = reinterpret_cast<uint32_t&>(_x);
94
+ if ((x & 0x7F800000) == 0x7F800000) {
95
+ return 0x80;
96
+ }
97
} else {
74
- x = reinterpret_cast<uint16_t&>(_x);
98
+ // if(__hisinf(x) || __hisnan(x))
99
+ if ((x & 0x7C00) == 0x7C00) {
100
+ return 0x80;
101
+ }
102
}
76
-
77
- uint32_t head, mantissa;
78
- int exponent, bias;
79
- uint32_t sign;
80
-
103
+ } else {
104
if (sizeof(T) == 4) {
82
- head = x & 0xFF800000;
83
- mantissa = x & 0x7FFFFF;
84
- exponent = (head >> 23) & 0xFF;
85
- sign = head >> 31;
86
- bias = 127;
105
+ if ((x & 0x7F800000) == 0x7F800000) {
106
+ return signed_inf + (mantissa != 0 ? 1 : 0);
107
+ }
108
} else {
88
- head = x & 0xFC00;
89
- mantissa = x & 0x3FF;
90
- exponent = (head >> 10) & 0x1F;
91
- sign = head >> 15;
92
- bias = 15;
109
+ if ((x & 0x7C00) == 0x7C00) {
110
+ return signed_inf + (mantissa != 0 ? 1 : 0);
111
+ }
112
}
94
-
95
- uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm);
96
-
97
- // Deal with inf and NaNs
98
- if (negative_zero_nan) {
99
- if (sizeof(T) == 4) {
100
- if ((x & 0x7F800000) == 0x7F800000) {
101
- return 0x80;
102
- }
103
- } else {
104
- // if(__hisinf(x) || __hisnan(x))
105
- if ((x & 0x7C00) == 0x7C00) {
106
- return 0x80;
107
- }
108
- }
109
- } else {
110
- if (sizeof(T) == 4) {
111
- if ((x & 0x7F800000) == 0x7F800000) {
112
- return signed_inf + (mantissa != 0 ? 1 : 0);
113
- }
114
- } else {
115
- if ((x & 0x7C00) == 0x7C00) {
116
- return signed_inf + (mantissa != 0 ? 1 : 0);
117
- }
118
- }
119
- }
120
- if (x == 0) {
121
- return 0;
122
- }
123
-
124
- // First need to check if it is normal or denorm as there is a difference of
125
- // implicit 1 Then need to adjust the exponent to align with the F8 exponent,
126
- // in the meanwhile, shift The mantissa. Then for stochastic rounding, add rng
127
- // to mantissa and truncate. And for RNE, no need to add rng. Then probably
128
- // need to check whether there is carry and adjust exponent and mantissa again
129
-
130
- // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent
131
- // bits
132
- const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0);
133
- const int f8_denormal_act_exponent = 1 - f8_bias; // actual exponent of f8 denormal
134
- // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias)
135
- // f8_exponent is the converted f8 exponent with bias encoding
136
- // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent,
137
- // the difference needs to be adjusted and mantissa shifted
138
- int act_exponent, f8_exponent, exponent_diff;
139
-
140
- if (exponent == 0) { // fp32/fp16 is in denormal.
141
- /* fp32 denormal is below 2^-127 so it is usually not a concern here, we
113
+ }
114
+ if (x == 0) {
115
+ return 0;
116
+ }
117
+
118
+ // First need to check if it is normal or denorm as there is a difference of
119
+ // implicit 1 Then need to adjust the exponent to align with the F8 exponent,
120
+ // in the meanwhile, shift The mantissa. Then for stochastic rounding, add rng
121
+ // to mantissa and truncate. And for RNE, no need to add rng. Then probably
122
+ // need to check whether there is carry and adjust exponent and mantissa again
123
+
124
+ // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent
125
+ // bits
126
+ const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0);
127
+ const int f8_denormal_act_exponent =
128
+ 1 - f8_bias; // actual exponent of f8 denormal
129
+ // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias)
130
+ // f8_exponent is the converted f8 exponent with bias encoding
131
+ // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent,
132
+ // the difference needs to be adjusted and mantissa shifted
133
+ int act_exponent, f8_exponent, exponent_diff;
134
+
135
+ if (exponent == 0) { // fp32/fp16 is in denormal.
136
+ /* fp32 denormal is below 2^-127 so it is usually not a concern here, we
137
mostly concern fp16 here. In this case, f8 is usually in denormal. But there
138
could be exceptions. fp16 denormal has exponent bias 15 while bf8 with NANOO has
139
exponent bias 16. It means that there are some numbers in fp16 denormal but they
140
are bf8 (NANOO) normals - smallest bf8 (NANOO) normal is 2^-15. fp16 numbers
141
where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 are bf8
142
(NANOO) normal. In this case, the fp16 mantissa should be shift left by 1 */
148
- act_exponent = exponent - bias + 1;
149
- exponent_diff = f8_denormal_act_exponent - act_exponent; // actual exponent is exponent-bias+1 as it is denormal
150
- } else { // fp32/fp16 is normal with implicit 1
151
- act_exponent = exponent - bias;
152
- if (act_exponent <= f8_denormal_act_exponent) {
153
- /* This is the case where fp32/fp16 is normal but it is in f8 denormal
154
- range. For example fp8 nanoo mode, denormal exponent is -7, but if the
155
- fp32/fp16 actual exponent is -7, it is actually larger due to the implicit 1,
156
- Therefore it needs to be adjust to -6 and mantissa shift right by 1.
157
- So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */
158
- exponent_diff = f8_denormal_act_exponent - act_exponent;
159
- } else { // both fp32/fp16 and f8 are in normal range
160
- exponent_diff = 0; // exponent_diff=0 does not mean there is no difference
161
- // for this case,
162
- // act_exponent could be larger. Just that it does not need shift mantissa
163
- }
164
- mantissa += (1 << mfmt); // Add the implicit 1 into mantissa
143
+ act_exponent = exponent - bias + 1;
144
+ exponent_diff =
145
+ f8_denormal_act_exponent -
146
+ act_exponent; // actual exponent is exponent-bias+1 as it is denormal
147
+ } else { // fp32/fp16 is normal with implicit 1
148
+ act_exponent = exponent - bias;
149
+ if (act_exponent <= f8_denormal_act_exponent) {
150
+ /* This is the case where fp32/fp16 is normal but it is in f8 denormal
151
+range. For example fp8 nanoo mode, denormal exponent is -7, but if the
152
+fp32/fp16 actual exponent is -7, it is actually larger due to the implicit 1,
153
+Therefore it needs to be adjust to -6 and mantissa shift right by 1.
154
+So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */
155
+ exponent_diff = f8_denormal_act_exponent - act_exponent;
156
+ } else { // both fp32/fp16 and f8 are in normal range
157
+ exponent_diff = 0; // exponent_diff=0 does not mean there is no
158
+ // difference for this case, act_exponent could be
159
+ // larger. Just that it does not need shift mantissa
160
}
166
-
167
- bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) ==
168
- static_cast<uint32_t>(1 << (mfmt - wm + exponent_diff - 1));
169
- /* This part is a bit tricky. The judgment of whether it is a tie needs to be
170
- done before we shift right as shift right could rip off some residual part
171
- and make something not midpoint look like midpoint. For example, the fp16
172
- number 0x1002 (0 00100 0000000010), it is larger than midpoint, but after
173
- shift right by 4 bits, it would look like midpoint.
161
+ mantissa += (1 << mfmt); // Add the implicit 1 into mantissa
162
+ }
163
+
164
+ bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) ==
165
+ static_cast<uint32_t>(1 << (mfmt - wm + exponent_diff - 1));
166
+ /* This part is a bit tricky. The judgment of whether it is a tie needs to be
167
+ done before we shift right as shift right could rip off some residual part
168
+ and make something not midpoint look like midpoint. For example, the fp16
169
+ number 0x1002 (0 00100 0000000010), it is larger than midpoint, but after
170
+ shift right by 4 bits, it would look like midpoint.
171
*/
172
176
- if (exponent_diff > 0) {
177
- mantissa >>= exponent_diff;
178
- } else if (exponent_diff == -1) {
179
- mantissa <<= -exponent_diff;
173
+ if (exponent_diff > 0) {
174
+ mantissa >>= exponent_diff;
175
+ } else if (exponent_diff == -1) {
176
+ mantissa <<= -exponent_diff;
177
+ }
178
+ bool implicit_one = mantissa & (1 << mfmt);
179
+ // if there is no implicit 1, it means the f8 is denormal and need to adjust
180
+ // to denorm exponent
181
+ f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ +
182
+ f8_bias - (implicit_one ? 0 : 1);
183
+
184
+ // Now we have the exponent and mantissa adjusted
185
+ uint32_t drop_mask = (1 << (mfmt - wm)) - 1;
186
+ bool odd = mantissa & (1 << (mfmt - wm)); // if the least significant bit
187
+ // that is not truncated is 1
188
+ mantissa +=
189
+ (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) &
190
+ drop_mask;
191
+
192
+ // Now we deal with overflow
193
+ if (f8_exponent == 0) {
194
+ if ((1 << mfmt) & mantissa) {
195
+ f8_exponent = 1; // denormal overflow to become normal, promote exponent
196
}
181
- bool implicit_one = mantissa & (1 << mfmt);
182
- // if there is no implicit 1, it means the f8 is denormal and need to adjust
183
- // to denorm exponent
184
- f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias - (implicit_one ? 0 : 1);
185
-
186
- // Now we have the exponent and mantissa adjusted
187
- uint32_t drop_mask = (1 << (mfmt - wm)) - 1;
188
- bool odd = mantissa & (1 << (mfmt - wm)); // if the least significant bit that
189
- // is not truncated is 1
190
- mantissa += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask;
191
-
192
- // Now we deal with overflow
193
- if (f8_exponent == 0) {
194
- if ((1 << mfmt) & mantissa) {
195
- f8_exponent = 1; // denormal overflow to become normal, promote exponent
196
- }
197
- } else {
198
- if ((1 << (mfmt + 1)) & mantissa) {
199
- mantissa >>= 1;
200
- f8_exponent++;
201
- }
197
+ } else {
198
+ if ((1 << (mfmt + 1)) & mantissa) {
199
+ mantissa >>= 1;
200
+ f8_exponent++;
201
}
202
+ }
203
204
- mantissa >>= (mfmt - wm);
205
-
206
- // above range: quantize to maximum possible float of the same sign
207
- const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2);
208
- if (f8_exponent > max_exp) {
209
- if (clip) {
210
- mantissa = (1 << wm) - 1;
211
- f8_exponent = max_exp;
212
- } else {
213
- return signed_inf;
214
- }
215
- }
204
+ mantissa >>= (mfmt - wm);
205
217
- if (f8_exponent == 0 && mantissa == 0) {
218
- return negative_zero_nan ? 0 : (sign << 7);
206
+ // above range: quantize to maximum possible float of the same sign
207
+ const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2);
208
+ if (f8_exponent > max_exp) {
209
+ if (clip) {
210
+ mantissa = (1 << wm) - 1;
211
+ f8_exponent = max_exp;
212
+ } else {
213
+ return signed_inf;
214
}
220
- mantissa &= (1 << wm) - 1;
221
- return (sign << 7) | (f8_exponent << wm) | mantissa;
215
+ }
216
+
217
+ if (f8_exponent == 0 && mantissa == 0) {
218
+ return negative_zero_nan ? 0 : (sign << 7);
219
+ }
220
+ mantissa &= (1 << wm) - 1;
221
+ return (sign << 7) | (f8_exponent << wm) | mantissa;
222
}
223
224
template <int we, int wm, typename T = float, bool negative_zero_nan = true>
225
-inline HIP_FP8_HOST_DEVICE T from_float8(uint8_t x)
226
-{
225
+inline HIP_FP8_HOST_DEVICE T from_float8(uint8_t x) {
226
#ifdef __HIPCC__
228
- constexpr bool is_half = std::is_same<T, _Float16>::value;
227
+ constexpr bool is_half = std::is_same<T, _Float16>::value;
228
#else
230
- constexpr bool is_half = false;
229
+ constexpr bool is_half = false;
230
#endif
232
- constexpr bool is_float = std::is_same<T, float>::value;
233
- static_assert(is_half || is_float, "only half and float are supported");
231
+ constexpr bool is_float = std::is_same<T, float>::value;
232
+ static_assert(is_half || is_float, "only half and float are supported");
233
235
- constexpr int weo = is_half ? 5 : 8;
236
- constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7);
234
+ constexpr int weo = is_half ? 5 : 8;
235
+ constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7);
236
238
- T fInf, fNegInf, fNaN, fNeg0;
237
+ T fInf, fNegInf, fNaN, fNeg0;
238
239
#ifdef __HIPCC__
241
- if (is_half) {
242
- const uint16_t ihInf = 0x7C00;
243
- const uint16_t ihNegInf = 0xFC00;
244
- const uint16_t ihNaN = 0x7C01;
245
- const uint16_t ihNeg0 = 0x8000;
246
- fInf = reinterpret_cast<const _Float16&>(ihInf);
247
- fNegInf = reinterpret_cast<const _Float16&>(ihNegInf);
248
- fNaN = reinterpret_cast<const _Float16&>(ihNaN);
249
- fNeg0 = reinterpret_cast<const _Float16&>(ihNeg0);
250
- } else
240
+ if (is_half) {
241
+ const uint16_t ihInf = 0x7C00;
242
+ const uint16_t ihNegInf = 0xFC00;
243
+ const uint16_t ihNaN = 0x7C01;
244
+ const uint16_t ihNeg0 = 0x8000;
245
+ fInf = reinterpret_cast<const _Float16&>(ihInf);
246
+ fNegInf = reinterpret_cast<const _Float16&>(ihNegInf);
247
+ fNaN = reinterpret_cast<const _Float16&>(ihNaN);
248
+ fNeg0 = reinterpret_cast<const _Float16&>(ihNeg0);
249
+ } else
250
#endif
252
- if (is_float) {
253
- const uint32_t ifInf = 0x7F800000;
254
- const uint32_t ifNegInf = 0xFF800000;
255
- const uint32_t ifNaN = 0x7F800001;
256
- const uint32_t ifNeg0 = 0x80000000;
257
- fInf = reinterpret_cast<const float&>(ifInf);
258
- fNegInf = reinterpret_cast<const float&>(ifNegInf);
259
- fNaN = reinterpret_cast<const float&>(ifNaN);
260
- fNeg0 = reinterpret_cast<const float&>(ifNeg0);
261
- }
262
-
263
- if (x == 0) {
264
- return 0;
265
- }
266
-
267
- uint32_t sign = x >> 7;
268
- uint32_t mantissa = x & ((1 << wm) - 1);
269
- int exponent = (x & 0x7F) >> wm;
270
- if (negative_zero_nan) {
271
- if (x == 0x80) {
272
- return fNaN;
273
- }
274
- } else {
275
- if (x == 0x80) {
276
- return fNeg0;
277
- }
278
- if (exponent == ((1 << we) - 1)) {
279
- return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN;
280
- }
281
- }
282
- typename std::conditional<sizeof(T) == 2, uint16_t, uint32_t>::type retval;
283
- if (we == 5 && is_half && !negative_zero_nan) {
284
- retval = x << 8;
285
- return reinterpret_cast<const T&>(retval);
251
+ if (is_float) {
252
+ const uint32_t ifInf = 0x7F800000;
253
+ const uint32_t ifNegInf = 0xFF800000;
254
+ const uint32_t ifNaN = 0x7F800001;
255
+ const uint32_t ifNeg0 = 0x80000000;
256
+ fInf = reinterpret_cast<const float&>(ifInf);
257
+ fNegInf = reinterpret_cast<const float&>(ifNegInf);
258
+ fNaN = reinterpret_cast<const float&>(ifNaN);
259
+ fNeg0 = reinterpret_cast<const float&>(ifNeg0);
260
+ }
261
+
262
+ if (x == 0) {
263
+ return 0;
264
+ }
265
+
266
+ uint32_t sign = x >> 7;
267
+ uint32_t mantissa = x & ((1 << wm) - 1);
268
+ int exponent = (x & 0x7F) >> wm;
269
+ if (negative_zero_nan) {
270
+ if (x == 0x80) {
271
+ return fNaN;
272
}
287
-
288
- const int exp_low_cutoff = (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0);
289
-
290
- // subnormal input
291
- if (exponent == 0) {
292
- // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above
293
- int sh = 1 + clz(mantissa) - (32 - wm);
294
- mantissa <<= sh;
295
- exponent += 1 - sh;
296
- mantissa &= ((1 << wm) - 1);
273
+ } else {
274
+ if (x == 0x80) {
275
+ return fNeg0;
276
}
298
- exponent += exp_low_cutoff - 1;
299
- mantissa <<= wmo - wm;
300
-
301
- // subnormal output (occurs when T=half, we=5, negative_zero_nan=true)
302
- if (exponent <= 0) {
303
- mantissa |= 1 << wmo;
304
- mantissa >>= 1 - exponent;
305
- exponent = 0;
306
- }
307
-
308
- if (sizeof(T) == 2) {
309
- retval = (sign << 15) | (exponent << 10) | mantissa;
310
- } else {
311
- retval = (sign << 31) | (exponent << 23) | mantissa;
277
+ if (exponent == ((1 << we) - 1)) {
278
+ return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN;
279
}
280
+ }
281
+ typename std::conditional<sizeof(T) == 2, uint16_t, uint32_t>::type retval;
282
+ if (we == 5 && is_half && !negative_zero_nan) {
283
+ retval = x << 8;
284
return reinterpret_cast<const T&>(retval);
285
+ }
286
+
287
+ const int exp_low_cutoff =
288
+ (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0);
289
+
290
+ // subnormal input
291
+ if (exponent == 0) {
292
+ // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above
293
+ int sh = 1 + clz(mantissa) - (32 - wm);
294
+ mantissa <<= sh;
295
+ exponent += 1 - sh;
296
+ mantissa &= ((1 << wm) - 1);
297
+ }
298
+ exponent += exp_low_cutoff - 1;
299
+ mantissa <<= wmo - wm;
300
+
301
+ // subnormal output (occurs when T=half, we=5, negative_zero_nan=true)
302
+ if (exponent <= 0) {
303
+ mantissa |= 1 << wmo;
304
+ mantissa >>= 1 - exponent;
305
+ exponent = 0;
306
+ }
307
+
308
+ if (sizeof(T) == 2) {
309
+ retval = (sign << 15) | (exponent << 10) | mantissa;
310
+ } else {
311
+ retval = (sign << 31) | (exponent << 23) | mantissa;
312
+ }
313
+ return reinterpret_cast<const T&>(retval);
314
}
315
316
-} // namespace hip_fp8_impl
316
+} // namespace hip_fp8_impl
csrc/quantization/fp8/amd/quant_utils.cuh
+356
-355
@@ -9,566 +9,567 @@
9
#include "../../../attention/dtype_float32.cuh"
10
#include "../../../attention/dtype_bfloat16.cuh"
11
12
-namespace vllm
13
-{
12
+namespace vllm {
13
#ifdef USE_ROCM
14
15
namespace fp8 {
17
-#ifdef ENABLE_FP8
16
+ #ifdef ENABLE_FP8
17
18
template <typename Tout, typename Tin>
20
-__inline__ __device__ Tout vec_conversion(const Tin& x)
21
-{
22
- return x;
19
+__inline__ __device__ Tout vec_conversion(const Tin& x) {
20
+ return x;
21
}
22
23
template <typename Tout, typename Tin>
26
-__inline__ __device__ Tout scaled_vec_conversion(const Tin& x, const float scale)
27
-{
28
- return x;
24
+__inline__ __device__ Tout scaled_vec_conversion(const Tin& x,
25
+ const float scale) {
26
+ return x;
27
}
28
29
// fp8 -> half
30
template <>
33
-__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
34
-{
35
- hip_fp8 f8{a, hip_fp8::from_bits()};
36
- __half_raw res;
37
- res.data = static_cast<float>(f8);
38
- return res.x;
31
+__inline__ __device__ uint16_t
32
+vec_conversion<uint16_t, uint8_t>(const uint8_t& a) {
33
+ hip_fp8 f8{a, hip_fp8::from_bits()};
34
+ __half_raw res;
35
+ res.data = static_cast<float>(f8);
36
+ return res.x;
37
}
38
39
// fp8x2 -> half2
40
template <>
43
-__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
44
-{
45
-#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
46
- const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
47
- union {
48
- __half2_raw h2r;
49
- uint32_t ui32;
50
- } tmp;
51
- tmp.h2r.x.data = f2[0];
52
- tmp.h2r.y.data = f2[1];
53
- return tmp.ui32;
54
-#else
55
- union {
56
- uint16_t u16[2];
57
- uint32_t u32;
58
- } tmp;
59
-
60
- tmp.u16[0] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a));
61
- tmp.u16[1] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U));
62
- return tmp.u32;
63
-#endif
41
+__inline__ __device__ uint32_t
42
+vec_conversion<uint32_t, uint16_t>(const uint16_t& a) {
43
+ #if defined(__HIP__MI300__) && \
44
+ defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
45
+ const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
46
+ union {
47
+ __half2_raw h2r;
48
+ uint32_t ui32;
49
+ } tmp;
50
+ tmp.h2r.x.data = f2[0];
51
+ tmp.h2r.y.data = f2[1];
52
+ return tmp.ui32;
53
+ #else
54
+ union {
55
+ uint16_t u16[2];
56
+ uint32_t u32;
57
+ } tmp;
58
+
59
+ tmp.u16[0] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a));
60
+ tmp.u16[1] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U));
61
+ return tmp.u32;
62
+ #endif
63
}
64
65
// fp8x4 -> half2x2
66
template <>
68
-__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
69
-{
70
- union {
71
- uint2 u32x2;
72
- uint32_t u32[2];
73
- } tmp;
74
- tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
75
- tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
76
- return tmp.u32x2;
67
+__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a) {
68
+ union {
69
+ uint2 u32x2;
70
+ uint32_t u32[2];
71
+ } tmp;
72
+ tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
73
+ tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
74
+ return tmp.u32x2;
75
}
76
77
// fp8x8 -> half2x4
78
template <>
81
-__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
82
-{
83
- union {
84
- uint4 u64x2;
85
- uint2 u64[2];
86
- } tmp;
87
- tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
88
- tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
89
- return tmp.u64x2;
79
+__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a) {
80
+ union {
81
+ uint4 u64x2;
82
+ uint2 u64[2];
83
+ } tmp;
84
+ tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
85
+ tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
86
+ return tmp.u64x2;
87
}
88
89
using __nv_bfloat16 = __hip_bfloat16;
90
91
// fp8 -> __nv_bfloat16
92
template <>
96
-__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
97
-{
98
- hip_fp8 f8{a, hip_fp8::from_bits()};
99
- float f{f8};
100
- return __float2bfloat16(f);
93
+__inline__ __device__ __nv_bfloat16
94
+vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a) {
95
+ hip_fp8 f8{a, hip_fp8::from_bits()};
96
+ float f{f8};
97
+ return __float2bfloat16(f);
98
}
99
100
using __nv_bfloat162 = __hip_bfloat162;
101
102
// fp8x2 -> __nv_bfloat162
103
template <>
107
-__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
108
-{
109
- __nv_bfloat162 res;
110
- res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
111
- res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
112
- return res;
104
+__inline__ __device__ __nv_bfloat162
105
+vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a) {
106
+ __nv_bfloat162 res;
107
+ res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
108
+ res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
109
+ return res;
110
}
111
112
// fp8x4 -> bf16_4_t
113
template <>
117
-__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
118
-{
119
- bf16_4_t res;
120
- res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
121
- res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
122
- return res;
114
+__inline__ __device__ bf16_4_t
115
+vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a) {
116
+ bf16_4_t res;
117
+ res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
118
+ res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
119
+ return res;
120
}
121
122
// fp8x8 -> bf16_8_t
123
template <>
127
-__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
128
-{
129
- bf16_4_t tmp1, tmp2;
130
- tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
131
- tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
132
- bf16_8_t res;
133
- res.x = tmp1.x;
134
- res.y = tmp1.y;
135
- res.z = tmp2.x;
136
- res.w = tmp2.y;
137
- return res;
124
+__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a) {
125
+ bf16_4_t tmp1, tmp2;
126
+ tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
127
+ tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
128
+ bf16_8_t res;
129
+ res.x = tmp1.x;
130
+ res.y = tmp1.y;
131
+ res.z = tmp2.x;
132
+ res.w = tmp2.y;
133
+ return res;
134
}
135
136
// fp8 -> float
137
template <>
142
-__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
143
-{
144
- hip_fp8 fp8{a, hip_fp8::from_bits()};
145
- return static_cast<float>(fp8);
138
+__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a) {
139
+ hip_fp8 fp8{a, hip_fp8::from_bits()};
140
+ return static_cast<float>(fp8);
141
}
142
143
// fp8x2 -> float2
144
template <>
150
-__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
151
-{
152
-#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
153
- float2 res;
154
- const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
155
- res.x = f2[0];
156
- res.y = f2[1];
157
- return res;
158
-#else
159
- float2 res;
160
- res.x = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a));
161
- res.y = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U));
162
- return res;
163
-#endif
145
+__inline__ __device__ float2
146
+vec_conversion<float2, uint16_t>(const uint16_t& a) {
147
+ #if defined(__HIP__MI300__) && \
148
+ defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
149
+ float2 res;
150
+ const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
151
+ res.x = f2[0];
152
+ res.y = f2[1];
153
+ return res;
154
+ #else
155
+ float2 res;
156
+ res.x = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a));
157
+ res.y = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U));
158
+ return res;
159
+ #endif
160
}
161
162
// fp8x4 -> float4
163
template <>
168
-__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
169
-{
170
- Float4_ res;
171
- res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
172
- res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
173
- return res;
164
+__inline__ __device__ Float4_
165
+vec_conversion<Float4_, uint32_t>(const uint32_t& a) {
166
+ Float4_ res;
167
+ res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
168
+ res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
169
+ return res;
170
}
171
172
// fp8x8 -> float8
173
template <>
178
-__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
179
-{
180
- Float4_ tmp1, tmp2;
181
- tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
182
- tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
183
- Float8_ res;
184
- res.x = tmp1.x;
185
- res.y = tmp1.y;
186
- res.z = tmp2.x;
187
- res.w = tmp2.y;
188
- return res;
174
+__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a) {
175
+ Float4_ tmp1, tmp2;
176
+ tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
177
+ tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
178
+ Float8_ res;
179
+ res.x = tmp1.x;
180
+ res.y = tmp1.y;
181
+ res.z = tmp2.x;
182
+ res.w = tmp2.y;
183
+ return res;
184
}
185
186
// half -> fp8
187
template <>
193
-__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
194
-{
195
- __half_raw tmp;
196
- tmp.x = a;
188
+__inline__ __device__ uint8_t
189
+vec_conversion<uint8_t, uint16_t>(const uint16_t& a) {
190
+ __half_raw tmp;
191
+ tmp.x = a;
192
198
- hip_fp8 f8{static_cast<float>(tmp.data)};
199
- return f8.data;
193
+ hip_fp8 f8{static_cast<float>(tmp.data)};
194
+ return f8.data;
195
}
196
197
// bf16 -> fp8
198
template <>
204
-__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
205
-{
206
- hip_fp8 res{__bfloat162float(a)};
207
- return res.data;
199
+__inline__ __device__ uint8_t
200
+vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a) {
201
+ hip_fp8 res{__bfloat162float(a)};
202
+ return res.data;
203
}
204
205
// float -> fp8
206
template <>
212
-__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
213
-{
214
- hip_fp8 f8(a);
215
- return f8.data;
207
+__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a) {
208
+ hip_fp8 f8(a);
209
+ return f8.data;
210
}
211
212
// fp8x4 -> float4
213
template <>
220
-__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
221
-{
222
- Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
223
- float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
224
- return res;
214
+__inline__ __device__ float4
215
+vec_conversion<float4, uint32_t>(const uint32_t& a) {
216
+ Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
217
+ float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
218
+ return res;
219
}
220
221
// float2 -> half2
222
template <>
229
-__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
230
-{
231
- union {
232
- half2 float16;
233
- uint32_t uint32;
234
- };
223
+__inline__ __device__ uint32_t
224
+vec_conversion<uint32_t, float2>(const float2& a) {
225
+ union {
226
+ half2 float16;
227
+ uint32_t uint32;
228
+ };
229
236
- float16 = __float22half2_rn(a);
237
- return uint32;
230
+ float16 = __float22half2_rn(a);
231
+ return uint32;
232
}
233
234
// Float4 -> half2x2
235
template <>
242
-__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
243
-{
244
- uint2 b;
245
- float2 val;
246
- val.x = a.x.x;
247
- val.y = a.x.y;
248
- b.x = vec_conversion<uint32_t, float2>(val);
236
+__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a) {
237
+ uint2 b;
238
+ float2 val;
239
+ val.x = a.x.x;
240
+ val.y = a.x.y;
241
+ b.x = vec_conversion<uint32_t, float2>(val);
242
250
- val.x = a.y.x;
251
- val.y = a.y.y;
252
- b.y = vec_conversion<uint32_t, float2>(val);
253
- return b;
243
+ val.x = a.y.x;
244
+ val.y = a.y.y;
245
+ b.y = vec_conversion<uint32_t, float2>(val);
246
+ return b;
247
}
248
249
// Float4 -> float4
250
template <>
258
-__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
259
-{
260
- float4 b;
261
- b.x = a.x.x;
262
- b.y = a.x.y;
263
- b.z = a.y.x;
264
- b.w = a.y.y;
265
- return b;
251
+__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a) {
252
+ float4 b;
253
+ b.x = a.x.x;
254
+ b.y = a.x.y;
255
+ b.z = a.y.x;
256
+ b.w = a.y.y;
257
+ return b;
258
}
259
260
// Float8 -> half2x4
261
template <>
270
-__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
271
-{
272
- uint4 b;
273
- b.x = vec_conversion<uint32_t, float2>(a.x);
274
- b.y = vec_conversion<uint32_t, float2>(a.y);
275
- b.z = vec_conversion<uint32_t, float2>(a.z);
276
- b.w = vec_conversion<uint32_t, float2>(a.w);
277
- return b;
262
+__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a) {
263
+ uint4 b;
264
+ b.x = vec_conversion<uint32_t, float2>(a.x);
265
+ b.y = vec_conversion<uint32_t, float2>(a.y);
266
+ b.z = vec_conversion<uint32_t, float2>(a.z);
267
+ b.w = vec_conversion<uint32_t, float2>(a.w);
268
+ return b;
269
}
270
271
// float2 -> bfloat162
272
template <>
282
-__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2& a)
283
-{
284
- __nv_bfloat162 b = __float22bfloat162_rn(a);
285
- return b;
273
+__inline__ __device__ __nv_bfloat162
274
+vec_conversion<__nv_bfloat162, float2>(const float2& a) {
275
+ __nv_bfloat162 b = __float22bfloat162_rn(a);
276
+ return b;
277
}
278
279
// Float4 -> bfloat162x2
280
template <>
290
-__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_& a)
291
-{
292
- bf16_4_t b;
293
- b.x = __float22bfloat162_rn(a.x);
294
- b.y = __float22bfloat162_rn(a.y);
295
- return b;
281
+__inline__ __device__ bf16_4_t
282
+vec_conversion<bf16_4_t, Float4_>(const Float4_& a) {
283
+ bf16_4_t b;
284
+ b.x = __float22bfloat162_rn(a.x);
285
+ b.y = __float22bfloat162_rn(a.y);
286
+ return b;
287
}
288
289
// Float8 -> bfloat162x4
290
template <>
300
-__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_& a)
301
-{
302
- bf16_8_t b;
303
- b.x = __float22bfloat162_rn(a.x);
304
- b.y = __float22bfloat162_rn(a.y);
305
- b.z = __float22bfloat162_rn(a.z);
306
- b.w = __float22bfloat162_rn(a.w);
307
- return b;
291
+__inline__ __device__ bf16_8_t
292
+vec_conversion<bf16_8_t, Float8_>(const Float8_& a) {
293
+ bf16_8_t b;
294
+ b.x = __float22bfloat162_rn(a.x);
295
+ b.y = __float22bfloat162_rn(a.y);
296
+ b.z = __float22bfloat162_rn(a.z);
297
+ b.w = __float22bfloat162_rn(a.w);
298
+ return b;
299
}
300
301
+/* Scaled and vectorized conversions, for data exchange between high and low
302
+ precision domains
303
311
-/* Scaled and vectorized conversions, for data exchange between high and low precision domains
312
-
313
- Convention of the scale in API, e.g: FP8_data = Quantization( High_Precision_data / scale )
314
- s.t.
315
- Quantize(HP / scale) => FP8
316
- Dequant(FP8) * scale => HP
304
+ Convention of the scale in API, e.g: FP8_data = Quantization(
305
+ High_Precision_data / scale ) s.t. Quantize(HP / scale) => FP8 Dequant(FP8) *
306
+ scale => HP
307
308
*/
309
310
// fp8 -> half
311
template <>
322
-__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(const uint8_t& a, const float scale)
323
-{
324
- hip_fp8 f8{a, hip_fp8::from_bits()};
325
- __half_raw res;
326
- res.data = static_cast<float>(f8) * scale;
327
- return res.x;
312
+__inline__ __device__ uint16_t
313
+scaled_vec_conversion<uint16_t, uint8_t>(const uint8_t& a, const float scale) {
314
+ hip_fp8 f8{a, hip_fp8::from_bits()};
315
+ __half_raw res;
316
+ res.data = static_cast<float>(f8) * scale;
317
+ return res.x;
318
}
319
320
// fp8x2 -> half2
321
template <>
332
-__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(const uint16_t& a, const float scale)
333
-{
334
-#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
335
- const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
336
- union {
337
- __half2_raw h2r;
338
- uint32_t ui32;
339
- } tmp;
340
- tmp.h2r.x.data = f2[0] * scale;
341
- tmp.h2r.y.data = f2[1] * scale;
342
- return tmp.ui32;
343
-#else
344
- union {
345
- uint16_t u16[2];
346
- uint32_t u32;
347
- } tmp;
348
-
349
- tmp.u16[0] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a), scale);
350
- tmp.u16[1] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
351
- return tmp.u32;
352
-#endif
322
+__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
323
+ const uint16_t& a, const float scale) {
324
+ #if defined(__HIP__MI300__) && \
325
+ defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
326
+ const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
327
+ union {
328
+ __half2_raw h2r;
329
+ uint32_t ui32;
330
+ } tmp;
331
+ tmp.h2r.x.data = f2[0] * scale;
332
+ tmp.h2r.y.data = f2[1] * scale;
333
+ return tmp.ui32;
334
+ #else
335
+ union {
336
+ uint16_t u16[2];
337
+ uint32_t u32;
338
+ } tmp;
339
+
340
+ tmp.u16[0] =
341
+ scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a), scale);
342
+ tmp.u16[1] = scaled_vec_conversion<uint16_t, uint8_t>(
343
+ static_cast<uint8_t>(a >> 8U), scale);
344
+ return tmp.u32;
345
+ #endif
346
}
347
348
// fp8x4 -> half2x2
349
template <>
357
-__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(const uint32_t& a, const float scale)
358
-{
359
- union {
360
- uint2 u32x2;
361
- uint32_t u32[2];
362
- } tmp;
363
- tmp.u32[0] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale);
364
- tmp.u32[1] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), scale);
365
- return tmp.u32x2;
350
+__inline__ __device__ uint2
351
+scaled_vec_conversion<uint2, uint32_t>(const uint32_t& a, const float scale) {
352
+ union {
353
+ uint2 u32x2;
354
+ uint32_t u32[2];
355
+ } tmp;
356
+ tmp.u32[0] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale);
357
+ tmp.u32[1] =
358
+ scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), scale);
359
+ return tmp.u32x2;
360
}
361
362
// fp8x8 -> half2x4
363
template <>
370
-__inline__ __device__ uint4 scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale)
371
-{
372
- union {
373
- uint4 u64x2;
374
- uint2 u64[2];
375
- } tmp;
376
- tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale);
377
- tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale);
378
- return tmp.u64x2;
364
+__inline__ __device__ uint4
365
+scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale) {
366
+ union {
367
+ uint4 u64x2;
368
+ uint2 u64[2];
369
+ } tmp;
370
+ tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale);
371
+ tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale);
372
+ return tmp.u64x2;
373
}
374
375
using __nv_bfloat16 = __hip_bfloat16;
376
377
// fp8 -> __nv_bfloat16
378
template <>
385
-__inline__ __device__ __nv_bfloat16 scaled_vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a, const float scale)
386
-{
387
- hip_fp8 f8{a, hip_fp8::from_bits()};
388
- float f{f8};
389
- return __float2bfloat16(f * scale);
379
+__inline__ __device__ __nv_bfloat16
380
+scaled_vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a,
381
+ const float scale) {
382
+ hip_fp8 f8{a, hip_fp8::from_bits()};
383
+ float f{f8};
384
+ return __float2bfloat16(f * scale);
385
}
386
387
using __nv_bfloat162 = __hip_bfloat162;
388
389
// fp8x2 -> __nv_bfloat162
390
template <>
396
-__inline__ __device__ __nv_bfloat162 scaled_vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a, const float scale)
397
-{
398
- __nv_bfloat162 res;
399
- res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale);
400
- res.y = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), scale);
401
- return res;
391
+__inline__ __device__ __nv_bfloat162
392
+scaled_vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a,
393
+ const float scale) {
394
+ __nv_bfloat162 res;
395
+ res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale);
396
+ res.y =
397
+ scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), scale);
398
+ return res;
399
}
400
401
// fp8x4 -> bf16_4_t
402
template <>
406
-__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a, const float scale)
407
-{
408
- bf16_4_t res;
409
- res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale);
410
- res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), scale);
411
- return res;
403
+__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
404
+ const uint32_t& a, const float scale) {
405
+ bf16_4_t res;
406
+ res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale);
407
+ res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U),
408
+ scale);
409
+ return res;
410
}
411
412
// fp8x8 -> bf16_8_t
413
template <>
416
-__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(const uint2& a, const float scale)
417
-{
418
- bf16_4_t tmp1, tmp2;
419
- tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale);
420
- tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale);
421
- bf16_8_t res;
422
- res.x = tmp1.x;
423
- res.y = tmp1.y;
424
- res.z = tmp2.x;
425
- res.w = tmp2.y;
426
- return res;
414
+__inline__ __device__ bf16_8_t
415
+scaled_vec_conversion<bf16_8_t, uint2>(const uint2& a, const float scale) {
416
+ bf16_4_t tmp1, tmp2;
417
+ tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale);
418
+ tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale);
419
+ bf16_8_t res;
420
+ res.x = tmp1.x;
421
+ res.y = tmp1.y;
422
+ res.z = tmp2.x;
423
+ res.w = tmp2.y;
424
+ return res;
425
}
426
427
// fp8 -> float
428
template <>
431
-__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(const uint8_t& a, const float scale)
432
-{
433
- hip_fp8 fp8{a, hip_fp8::from_bits()};
434
- return static_cast<float>(fp8) * scale;
429
+__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
430
+ const uint8_t& a, const float scale) {
431
+ hip_fp8 fp8{a, hip_fp8::from_bits()};
432
+ return static_cast<float>(fp8) * scale;
433
}
434
435
// fp8x2 -> float2
436
template <>
439
-__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(const uint16_t& a, const float scale)
440
-{
441
-#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
442
- float2 res;
443
- const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
444
- res.x = f2[0] * scale;
445
- res.y = f2[1] * scale;
446
- return res;
447
-#else
448
- float2 res;
449
- res.x = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a), scale);
450
- res.y = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
451
- return res;
452
-#endif
437
+__inline__ __device__ float2
438
+scaled_vec_conversion<float2, uint16_t>(const uint16_t& a, const float scale) {
439
+ #if defined(__HIP__MI300__) && \
440
+ defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
441
+ float2 res;
442
+ const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
443
+ res.x = f2[0] * scale;
444
+ res.y = f2[1] * scale;
445
+ return res;
446
+ #else
447
+ float2 res;
448
+ res.x = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a), scale);
449
+ res.y = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U),
450
+ scale);
451
+ return res;
452
+ #endif
453
}
454
455
// fp8x4 -> float4
456
template <>
457
-__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(const uint32_t& a, const float scale)
458
-{
459
- Float4_ res;
460
- res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale);
461
- res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale);
462
- return res;
457
+__inline__ __device__ Float4_
458
+scaled_vec_conversion<Float4_, uint32_t>(const uint32_t& a, const float scale) {
459
+ Float4_ res;
460
+ res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale);
461
+ res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale);
462
+ return res;
463
}
464
465
// fp8x8 -> float8
466
template <>
467
-__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(const uint2& a, const float scale)
468
-{
469
- Float4_ tmp1, tmp2;
470
- tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale);
471
- tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale);
472
- Float8_ res;
473
- res.x = tmp1.x;
474
- res.y = tmp1.y;
475
- res.z = tmp2.x;
476
- res.w = tmp2.y;
477
- return res;
467
+__inline__ __device__ Float8_
468
+scaled_vec_conversion<Float8_, uint2>(const uint2& a, const float scale) {
469
+ Float4_ tmp1, tmp2;
470
+ tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale);
471
+ tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale);
472
+ Float8_ res;
473
+ res.x = tmp1.x;
474
+ res.y = tmp1.y;
475
+ res.z = tmp2.x;
476
+ res.w = tmp2.y;
477
+ return res;
478
}
479
480
-
480
/* Quantize(HP / scale) => FP8 */
481
482
// TODO(Hai): vectorized to add
483
484
// half -> fp8
485
template <>
487
-__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(const uint16_t& a, const float scale)
488
-{
489
- __half_raw tmp;
490
- tmp.x = a;
486
+__inline__ __device__ uint8_t
487
+scaled_vec_conversion<uint8_t, uint16_t>(const uint16_t& a, const float scale) {
488
+ __half_raw tmp;
489
+ tmp.x = a;
490
492
- hip_fp8 f8{static_cast<float>(tmp.data)/scale};
493
- return f8.data;
491
+ hip_fp8 f8{static_cast<float>(tmp.data) / scale};
492
+ return f8.data;
493
}
494
495
// bf16 -> fp8
496
template <>
498
-__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a, const float scale)
499
-{
500
- hip_fp8 res{__bfloat162float(a)/scale};
501
- return res.data;
497
+__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
498
+ const __nv_bfloat16& a, const float scale) {
499
+ hip_fp8 res{__bfloat162float(a) / scale};
500
+ return res.data;
501
}
502
503
// float -> fp8
504
template <>
506
-__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(const float& a, const float scale)
507
-{
508
- hip_fp8 f8(a/scale);
509
- return f8.data;
505
+__inline__ __device__ uint8_t
506
+scaled_vec_conversion<uint8_t, float>(const float& a, const float scale) {
507
+ hip_fp8 f8(a / scale);
508
+ return f8.data;
509
}
510
511
// fp8x4 -> float4
512
template <>
514
-__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(const uint32_t& a, const float scale)
515
-{
516
- Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale);
517
- float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
518
- return res;
513
+__inline__ __device__ float4
514
+scaled_vec_conversion<float4, uint32_t>(const uint32_t& a, const float scale) {
515
+ Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale);
516
+ float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
517
+ return res;
518
}
520
-#endif // ENABLE_FP8
519
+ #endif // ENABLE_FP8
520
521
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
523
-__inline__ __device__ Tout convert(const Tin &x) {
524
-#ifdef ENABLE_FP8
522
+__inline__ __device__ Tout convert(const Tin& x) {
523
+ #ifdef ENABLE_FP8
524
if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
525
return vec_conversion<Tout, Tin>(x);
526
}
528
-#endif
527
+ #endif
528
assert(false);
529
}
530
531
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
533
-__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
534
-#ifdef ENABLE_FP8
532
+__inline__ __device__ Tout scaled_convert(const Tin& x, const float scale) {
533
+ #ifdef ENABLE_FP8
534
if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
535
return scaled_vec_conversion<Tout, Tin>(x, scale);
536
}
538
-#endif
537
+ #endif
538
assert(false);
539
}
540
542
-// The following macro is used to dispatch the conversion function based on the
543
-// data type of the key and value cache. The FN is a macro that calls a function
544
-// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
545
-#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546
- if (KV_DTYPE == "auto") { \
547
- if (SRC_DTYPE == at::ScalarType::Float) { \
548
- FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549
- } else if (SRC_DTYPE == at::ScalarType::Half) { \
550
- FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551
- } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
552
- FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553
- } else { \
554
- TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555
- } \
556
- } else { \
557
- if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
541
+ // The following macro is used to dispatch the conversion function based on
542
+ // the data type of the key and value cache. The FN is a macro that calls a
543
+ // function with template<typename scalar_t, typename cache_t,
544
+ // Fp8KVCacheDataType kv_dt>.
545
+ #define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546
+ if (KV_DTYPE == "auto") { \
547
if (SRC_DTYPE == at::ScalarType::Float) { \
559
- FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548
+ FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549
} else if (SRC_DTYPE == at::ScalarType::Half) { \
561
- FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
550
+ FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551
} else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563
- FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
552
+ FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553
} else { \
554
TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555
} \
556
} else { \
568
- TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
569
- } \
570
- }
557
+ if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
558
+ if (SRC_DTYPE == at::ScalarType::Float) { \
559
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
560
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
561
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
562
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
564
+ } else { \
565
+ TORCH_CHECK(false, \
566
+ "Unsupported input type of kv cache: ", SRC_DTYPE); \
567
+ } \
568
+ } else { \
569
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
570
+ } \
571
+ }
572
572
-} // fp8
573
-#endif // USE_ROCM
574
-} // namespace vllm
573
+} // namespace fp8
574
+#endif // USE_ROCM
575
+} // namespace vllm
csrc/quantization/fp8/common.cu
+38
-49
@@ -10,17 +10,20 @@
10
namespace vllm {
11
12
__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
13
- float old;
14
- old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) :
15
- __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value)));
13
+ float old;
14
+ old = (value >= 0)
15
+ ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
16
+ : __uint_as_float(
17
+ atomicMin((unsigned int*)addr, __float_as_uint(value)));
18
17
- return old;
19
+ return old;
20
}
21
22
#define FP8_E4M3_MAX std::numeric_limits<c10::Float8_e4m3fn>::max()
23
22
-template<typename scalar_t>
23
-__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(const scalar_t val, const float scale) {
24
+template <typename scalar_t>
25
+__device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(
26
+ const scalar_t val, const float scale) {
27
float x = static_cast<float>(val) / scale;
28
float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
29
return static_cast<c10::Float8_e4m3fn>(r);
@@ -32,11 +35,10 @@ __device__ __forceinline__ c10::Float8_e4m3fn scaled_fp8_conversion(const scalar
35
// So to get the right answer, *scale needs to be initialized to
36
// a value <= 0.0 and we need to wait for all thread blocks to
37
// finish before consuming *scale.
35
-template<typename scalar_t>
36
-__global__ void segmented_max_reduction(
37
- float* __restrict__ scale,
38
- const scalar_t* __restrict__ input,
39
- int64_t num_elems) {
38
+template <typename scalar_t>
39
+__global__ void segmented_max_reduction(float* __restrict__ scale,
40
+ const scalar_t* __restrict__ input,
41
+ int64_t num_elems) {
42
__shared__ float cache[1024];
43
int i = blockDim.x * blockIdx.x + threadIdx.x;
44
@@ -56,7 +58,7 @@ __global__ void segmented_max_reduction(
58
int ib = blockDim.x / 2;
59
while (ib != 0) {
60
if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
59
- cache[threadIdx.x] = cache[threadIdx.x + ib];
61
+ cache[threadIdx.x] = cache[threadIdx.x + ib];
62
}
63
__syncthreads();
64
ib /= 2;
@@ -64,16 +66,16 @@ __global__ void segmented_max_reduction(
66
// Finally, since cache[0] contains the maximum for this thread block,
67
// atomically write the max to the target location
68
if (threadIdx.x == 0) {
67
- atomicMaxFloat(scale, cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
69
+ atomicMaxFloat(scale,
70
+ cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
71
}
72
}
73
71
-template<typename scalar_t>
72
-__global__ void scaled_fp8_quant_kernel(
73
- c10::Float8_e4m3fn* __restrict__ out,
74
- const scalar_t* __restrict__ input,
75
- const float* __restrict__ scale,
76
- int64_t num_elems) {
74
+template <typename scalar_t>
75
+__global__ void scaled_fp8_quant_kernel(c10::Float8_e4m3fn* __restrict__ out,
76
+ const scalar_t* __restrict__ input,
77
+ const float* __restrict__ scale,
78
+ int64_t num_elems) {
79
int i = blockDim.x * blockIdx.x + threadIdx.x;
80
while (i < num_elems) {
81
out[i] = scaled_fp8_conversion(input[i], *scale);
@@ -81,12 +83,11 @@ __global__ void scaled_fp8_quant_kernel(
83
}
84
}
85
84
-} // namespace vllm
86
+} // namespace vllm
87
86
-void static_scaled_fp8_quant(
87
- torch::Tensor& out, // [..., d]
88
- torch::Tensor& input, // [..., d]
89
- torch::Tensor& scale) // [1]
88
+void static_scaled_fp8_quant(torch::Tensor& out, // [..., d]
89
+ torch::Tensor& input, // [..., d]
90
+ torch::Tensor& scale) // [1]
91
{
92
int64_t num_tokens = input.numel() / input.size(-1);
93
int64_t num_elems = input.numel();
@@ -95,21 +96,16 @@ void static_scaled_fp8_quant(
96
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
97
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
98
VLLM_DISPATCH_FLOATING_TYPES(
98
- input.scalar_type(),
99
- "scaled_fp8_quant_kernel",
100
- [&] {
101
- vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
102
- out.data_ptr<c10::Float8_e4m3fn>(),
103
- input.data_ptr<scalar_t>(),
104
- scale.data_ptr<float>(),
105
- num_elems);
99
+ input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
100
+ vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
101
+ out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
102
+ scale.data_ptr<float>(), num_elems);
103
});
104
}
105
109
-void dynamic_scaled_fp8_quant(
110
- torch::Tensor& out, // [..., d]
111
- torch::Tensor& input, // [..., d]
112
- torch::Tensor& scale) // [1]
106
+void dynamic_scaled_fp8_quant(torch::Tensor& out, // [..., d]
107
+ torch::Tensor& input, // [..., d]
108
+ torch::Tensor& scale) // [1]
109
{
110
int64_t num_tokens = input.numel() / input.size(-1);
111
int64_t num_elems = input.numel();
@@ -118,18 +114,11 @@ void dynamic_scaled_fp8_quant(
114
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
115
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
116
VLLM_DISPATCH_FLOATING_TYPES(
121
- input.scalar_type(),
122
- "scaled_fp8_quant_kernel",
123
- [&] {
124
- vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
125
- scale.data_ptr<float>(),
126
- input.data_ptr<scalar_t>(),
127
- num_elems);
128
- vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
129
- out.data_ptr<c10::Float8_e4m3fn>(),
130
- input.data_ptr<scalar_t>(),
131
- scale.data_ptr<float>(),
132
- num_elems);
117
+ input.scalar_type(), "scaled_fp8_quant_kernel", [&] {
118
+ vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
119
+ scale.data_ptr<float>(), input.data_ptr<scalar_t>(), num_elems);
120
+ vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
121
+ out.data_ptr<c10::Float8_e4m3fn>(), input.data_ptr<scalar_t>(),
122
+ scale.data_ptr<float>(), num_elems);
123
});
124
}
135
-
csrc/quantization/fp8/nvidia/quant_utils.cuh
+70
-68
@@ -10,9 +10,9 @@ namespace vllm {
10
#ifndef USE_ROCM
11
12
namespace fp8 {
13
-#ifdef ENABLE_FP8
13
+ #ifdef ENABLE_FP8
14
15
-#if 0 // Disable the following code to reduce the binary size.
15
+ #if 0 // Disable the following code to reduce the binary size.
16
template <typename Tout, typename Tin>
17
__inline__ __device__ Tout
18
vec_conversion(const Tin &x, const __nv_fp8_interpretation_t fp8_type) {
@@ -177,13 +177,13 @@ __inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(
177
template <>
178
__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(
179
const __nv_bfloat16 &a, const __nv_fp8_interpretation_t fp8_type) {
180
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
180
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
181
assert(false);
182
-#else
182
+ #else
183
__nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(
184
__nv_bfloat16_raw(a), __NV_SATFINITE, fp8_type);
185
return (uint8_t)res;
186
-#endif
186
+ #endif
187
}
188
189
// float -> fp8
@@ -276,7 +276,7 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
276
from_float(b, a);
277
return b;
278
}
279
-#endif
279
+ #endif
280
281
/* Scaled and vectorized conversions, for data exchange between high and low
282
precision domains Convention of the scale in API, e.g: FP8_data =
@@ -286,14 +286,14 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
286
287
template <typename Tout, typename Tin>
288
__inline__ __device__ Tout scaled_vec_conversion(
289
- const Tin &x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
289
+ const Tin& x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
290
return x;
291
}
292
293
// fp8 -> half
294
template <>
295
__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
296
- const uint8_t &a, const float scale,
296
+ const uint8_t& a, const float scale,
297
const __nv_fp8_interpretation_t fp8_type) {
298
__half_raw tmp = __nv_cvt_fp8_to_halfraw(a, fp8_type);
299
return float_to_half(half_to_float(tmp.x) * scale);
@@ -302,7 +302,7 @@ __inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
302
// fp8x2 -> half2
303
template <>
304
__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
305
- const uint16_t &a, const float scale,
305
+ const uint16_t& a, const float scale,
306
const __nv_fp8_interpretation_t fp8_type) {
307
union {
308
uint16_t u16[2];
@@ -317,7 +317,7 @@ __inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
317
// fp8x4 -> half2x2
318
template <>
319
__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
320
- const uint32_t &a, const float scale,
320
+ const uint32_t& a, const float scale,
321
const __nv_fp8_interpretation_t fp8_type) {
322
union {
323
uint2 u32x2;
@@ -333,7 +333,7 @@ __inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
333
// fp8x8 -> half2x4
334
template <>
335
__inline__ __device__ uint4
336
-scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
336
+scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale,
337
const __nv_fp8_interpretation_t fp8_type) {
338
union {
339
uint4 u64x2;
@@ -348,7 +348,7 @@ scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
348
template <>
349
__inline__ __device__ __nv_bfloat16
350
scaled_vec_conversion<__nv_bfloat16, uint8_t>(
351
- const uint8_t &a, const float scale,
351
+ const uint8_t& a, const float scale,
352
const __nv_fp8_interpretation_t fp8_type) {
353
// Note there is no direct convert function from fp8 to bf16.
354
// fp8 -> half
@@ -362,7 +362,7 @@ scaled_vec_conversion<__nv_bfloat16, uint8_t>(
362
template <>
363
__inline__ __device__ __nv_bfloat162
364
scaled_vec_conversion<__nv_bfloat162, uint16_t>(
365
- const uint16_t &a, const float scale,
365
+ const uint16_t& a, const float scale,
366
const __nv_fp8_interpretation_t fp8_type) {
367
__nv_bfloat162 res;
368
res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale,
@@ -375,7 +375,7 @@ scaled_vec_conversion<__nv_bfloat162, uint16_t>(
375
// fp8x4 -> bf16_4_t
376
template <>
377
__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
378
- const uint32_t &a, const float scale,
378
+ const uint32_t& a, const float scale,
379
const __nv_fp8_interpretation_t fp8_type) {
380
bf16_4_t res;
381
res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale,
@@ -388,7 +388,7 @@ __inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
388
// fp8x8 -> bf16_8_t
389
template <>
390
__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
391
- const uint2 &a, const float scale,
391
+ const uint2& a, const float scale,
392
const __nv_fp8_interpretation_t fp8_type) {
393
bf16_4_t tmp1, tmp2;
394
tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale, fp8_type);
@@ -404,9 +404,8 @@ __inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
404
// fp8 -> float
405
template <>
406
__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
407
- const uint8_t &a, const float scale,
407
+ const uint8_t& a, const float scale,
408
const __nv_fp8_interpretation_t fp8_type) {
409
-
409
// fp8 -> half
410
__half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
411
uint16_t tmp = res.x;
@@ -418,7 +417,7 @@ __inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
417
// fp8x2 -> float2
418
template <>
419
__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
421
- const uint16_t &a, const float scale,
420
+ const uint16_t& a, const float scale,
421
const __nv_fp8_interpretation_t fp8_type) {
422
// fp8x2 -> half2
423
uint32_t tmp = scaled_vec_conversion<uint32_t, uint16_t>(a, scale, fp8_type);
@@ -429,7 +428,7 @@ __inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
428
// fp8x4 -> float4
429
template <>
430
__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
432
- const uint32_t &a, const float scale,
431
+ const uint32_t& a, const float scale,
432
const __nv_fp8_interpretation_t fp8_type) {
433
Float4_ res;
434
res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale, fp8_type);
@@ -441,7 +440,7 @@ __inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
440
// fp8x8 -> float8
441
template <>
442
__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
444
- const uint2 &a, const float scale,
443
+ const uint2& a, const float scale,
444
const __nv_fp8_interpretation_t fp8_type) {
445
Float4_ tmp1, tmp2;
446
tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale, fp8_type);
@@ -457,7 +456,7 @@ __inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
456
// half -> fp8
457
template <>
458
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
460
- const uint16_t &a, const float scale,
459
+ const uint16_t& a, const float scale,
460
const __nv_fp8_interpretation_t fp8_type) {
461
__nv_fp8_storage_t res =
462
__nv_cvt_float_to_fp8(half_to_float(a) / scale, __NV_SATFINITE, fp8_type);
@@ -467,21 +466,21 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
466
// bf16 -> fp8
467
template <>
468
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
470
- const __nv_bfloat16 &a, const float scale,
469
+ const __nv_bfloat16& a, const float scale,
470
const __nv_fp8_interpretation_t fp8_type) {
472
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
471
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
472
assert(false);
474
-#else
473
+ #else
474
__nv_fp8_storage_t res = __nv_cvt_float_to_fp8(__bfloat162float(a) / scale,
475
__NV_SATFINITE, fp8_type);
476
return (uint8_t)res;
478
-#endif
477
+ #endif
478
}
479
480
// float -> fp8
481
template <>
482
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
484
- const float &a, const float scale,
483
+ const float& a, const float scale,
484
const __nv_fp8_interpretation_t fp8_type) {
485
__nv_fp8_storage_t res =
486
__nv_cvt_float_to_fp8(a / scale, __NV_SATFINITE, fp8_type);
@@ -491,78 +490,81 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
490
// fp8x4 -> float4
491
template <>
492
__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(
494
- const uint32_t &a, const float scale,
493
+ const uint32_t& a, const float scale,
494
const __nv_fp8_interpretation_t fp8_type) {
495
Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale, fp8_type);
496
float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
497
return res;
498
}
500
-#endif // ENABLE_FP8
499
+ #endif // ENABLE_FP8
500
501
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
503
-__inline__ __device__ Tout convert(const Tin &x) {
504
-#if 0 // Disable the following code to reduce the binary size.
502
+__inline__ __device__ Tout convert(const Tin& x) {
503
+ #if 0 // Disable the following code to reduce the binary size.
504
if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
505
return vec_conversion<Tout, Tin>(x, __NV_E4M3);
506
} else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
507
return vec_conversion<Tout, Tin>(x, __NV_E5M2);
508
}
510
-#endif
509
+ #endif
510
assert(false);
511
}
512
513
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
515
-__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
516
-#ifdef ENABLE_FP8
514
+__inline__ __device__ Tout scaled_convert(const Tin& x, const float scale) {
515
+ #ifdef ENABLE_FP8
516
if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
517
return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E4M3);
518
} else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
519
return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E5M2);
520
}
522
-#endif
521
+ #endif
522
assert(false);
523
}
524
526
-// The following macro is used to dispatch the conversion function based on the
527
-// data type of the key and value cache. The FN is a macro that calls a function
528
-// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
529
-#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530
- if (KV_DTYPE == "auto") { \
531
- if (SRC_DTYPE == at::ScalarType::Float) { \
532
- FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533
- } else if (SRC_DTYPE == at::ScalarType::Half) { \
534
- FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535
- } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
536
- FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537
- } else { \
538
- TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539
- } \
540
- } else { \
541
- if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
525
+ // The following macro is used to dispatch the conversion function based on
526
+ // the data type of the key and value cache. The FN is a macro that calls a
527
+ // function with template<typename scalar_t, typename cache_t,
528
+ // Fp8KVCacheDataType kv_dt>.
529
+ #define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530
+ if (KV_DTYPE == "auto") { \
531
if (SRC_DTYPE == at::ScalarType::Float) { \
543
- FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
532
+ FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533
} else if (SRC_DTYPE == at::ScalarType::Half) { \
545
- FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
534
+ FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535
} else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547
- FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
536
+ FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537
} else { \
538
TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539
} \
551
- } else if (KV_DTYPE == "fp8_e5m2") { \
552
- if (SRC_DTYPE == at::ScalarType::Float) { \
553
- FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
554
- } else if (SRC_DTYPE == at::ScalarType::Half) { \
555
- FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
556
- } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
557
- FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
540
+ } else { \
541
+ if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
542
+ if (SRC_DTYPE == at::ScalarType::Float) { \
543
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
544
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
545
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
546
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548
+ } else { \
549
+ TORCH_CHECK(false, \
550
+ "Unsupported input type of kv cache: ", SRC_DTYPE); \
551
+ } \
552
+ } else if (KV_DTYPE == "fp8_e5m2") { \
553
+ if (SRC_DTYPE == at::ScalarType::Float) { \
554
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
555
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
556
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
557
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
558
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
559
+ } else { \
560
+ TORCH_CHECK(false, \
561
+ "Unsupported input type of kv cache: ", SRC_DTYPE); \
562
+ } \
563
} else { \
559
- TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
564
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
565
} \
561
- } else { \
562
- TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
563
- } \
564
- }
566
+ }
567
566
-} // namespace fp8
567
-#endif // not USE_ROCM
568
-} // namespace vllm
568
+} // namespace fp8
569
+#endif // not USE_ROCM
570
+} // namespace vllm
csrc/quantization/gptq/compat.cuh
+35
-35
@@ -9,54 +9,54 @@ namespace vllm {
9
namespace gptq {
10
// atomicAdd for half types, to support CC < 7.x
11
12
-__device__ __forceinline__ void atomicAdd_half(half* address, half val)
13
-{
14
- unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));
15
- unsigned int old = *address_as_ui;
16
- unsigned int assumed;
12
+__device__ __forceinline__ void atomicAdd_half(half* address, half val) {
13
+ unsigned int* address_as_ui =
14
+ (unsigned int*)((char*)address - ((size_t)address & 2));
15
+ unsigned int old = *address_as_ui;
16
+ unsigned int assumed;
17
18
- do
19
- {
20
- assumed = old;
21
- __half_raw hsum;
22
- hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
23
- half tmpres = __hadd(hsum, val);
24
- hsum = __half_raw(tmpres);
25
- old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;
26
- old = atomicCAS(address_as_ui, assumed, old);
27
- }
28
- while (assumed != old);
18
+ do {
19
+ assumed = old;
20
+ __half_raw hsum;
21
+ hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
22
+ half tmpres = __hadd(hsum, val);
23
+ hsum = __half_raw(tmpres);
24
+ old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16)
25
+ : (old & 0xffff0000) | hsum.x;
26
+ old = atomicCAS(address_as_ui, assumed, old);
27
+ } while (assumed != old);
28
}
29
30
// atomicAdd for half2 types
31
33
-__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)
34
-{
35
- unsigned int* address_as_ui = (unsigned int*)address;
36
- unsigned int old = *address_as_ui;
37
- unsigned int assumed;
38
- do
39
- {
40
- assumed = old;
41
- half2 old_val = *((half2*)&old);
42
- half2 new_val = __hadd2(old_val, val);
43
- old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
44
- }
45
- while (assumed != old);
32
+__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) {
33
+ unsigned int* address_as_ui = (unsigned int*)address;
34
+ unsigned int old = *address_as_ui;
35
+ unsigned int assumed;
36
+ do {
37
+ assumed = old;
38
+ half2 old_val = *((half2*)&old);
39
+ half2 new_val = __hadd2(old_val, val);
40
+ old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
41
+ } while (assumed != old);
42
}
43
44
//
45
46
#if defined(__CUDA_ARCH__) || defined(USE_ROCM)
51
-#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
47
+ #if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
48
53
-__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }
49
+__device__ __forceinline__ void atomicAdd(half* address, half val) {
50
+ atomicAdd_half(address, val);
51
+}
52
55
-#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
56
-__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }
57
-#endif
53
+ #if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
54
+__device__ __forceinline__ void atomicAdd(half2* address, half2 val) {
55
+ atomicAdd_half2(address, val);
56
+}
57
+ #endif
58
59
-#endif
59
+ #endif
60
#endif
61
62
} // namespace gptq
csrc/quantization/gptq/matrix_view.cuh
+262
-241
@@ -1,5 +1,6 @@
1
/*
2
-Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turboderp/exllama
2
+Adapted from https://github.com/turboderp/exllamav2 and
3
+https://github.com/turboderp/exllama
4
*/
5
6
#ifndef _matrix_view_cuh
@@ -13,260 +14,280 @@ Adapted from https://github.com/turboderp/exllamav2 and https://github.com/turbo
14
namespace vllm {
15
namespace gptq {
16
16
-class MatrixView_half
17
-{
18
-public:
19
- const half* data;
20
- const int height;
21
- const int width;
22
-
23
- __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)
24
- : data(data), height(height), width(width)
25
- { }
26
-
27
- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
28
- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
29
- __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }
30
- __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }
31
-
32
- __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const
33
- {
34
- half2* ptr = (half2*) item_ptr(row, column);
35
- half2 i01 = ptr[0];
36
- half2 i23 = ptr[1];
37
- items[0] = __low2half(i01);
38
- items[1] = __high2half(i01);
39
- items[2] = __low2half(i23);
40
- items[3] = __high2half(i23);
41
- }
42
- __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const
43
- {
44
- half2* ptr = (half2*)item_ptr(row, column);
45
- half2 i01 = ptr[0];
46
- half2 i23 = ptr[1];
47
- items[0] = __half2float(__low2half(i01));
48
- items[1] = __half2float(__high2half(i01));
49
- items[2] = __half2float(__low2half(i23));
50
- items[3] = __half2float(__high2half(i23));
51
- }
52
-
53
- __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const
54
- {
55
- half2* ptr = (half2*)item_ptr(row, column);
56
- half2 i01 = ptr[0];
57
- half2 i23 = ptr[1];
58
- items[0] = __half2half2(__low2half(i01));
59
- items[1] = __half2half2(__high2half(i01));
60
- items[2] = __half2half2(__low2half(i23));
61
- items[3] = __half2half2(__high2half(i23));
62
- }
17
+class MatrixView_half {
18
+ public:
19
+ const half* data;
20
+ const int height;
21
+ const int width;
22
+
23
+ __device__ __forceinline__ MatrixView_half(const half* data, const int height,
24
+ const int width)
25
+ : data(data), height(height), width(width) {}
26
+
27
+ __device__ __forceinline__ half item(int row, int column) const {
28
+ return data[row * width + column];
29
+ }
30
+ __device__ __forceinline__ half2 item_half2(int row, int column) const {
31
+ return ((half2*)data)[(row * width + column) / 2];
32
+ }
33
+ __device__ __forceinline__ half2 item_half2half2(int row, int column) const {
34
+ return __half2half2(data[row * width + column]);
35
+ }
36
+ __device__ __forceinline__ const half* item_ptr(int row, int column) const {
37
+ return &data[row * width + column];
38
+ }
39
+
40
+ __device__ __forceinline__ void item4(half (&items)[4], int row,
41
+ int column) const {
42
+ half2* ptr = (half2*)item_ptr(row, column);
43
+ half2 i01 = ptr[0];
44
+ half2 i23 = ptr[1];
45
+ items[0] = __low2half(i01);
46
+ items[1] = __high2half(i01);
47
+ items[2] = __low2half(i23);
48
+ items[3] = __high2half(i23);
49
+ }
50
+ __device__ __forceinline__ void item4_f(float (&items)[4], int row,
51
+ int column) const {
52
+ half2* ptr = (half2*)item_ptr(row, column);
53
+ half2 i01 = ptr[0];
54
+ half2 i23 = ptr[1];
55
+ items[0] = __half2float(__low2half(i01));
56
+ items[1] = __half2float(__high2half(i01));
57
+ items[2] = __half2float(__low2half(i23));
58
+ items[3] = __half2float(__high2half(i23));
59
+ }
60
+
61
+ __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row,
62
+ int column) const {
63
+ half2* ptr = (half2*)item_ptr(row, column);
64
+ half2 i01 = ptr[0];
65
+ half2 i23 = ptr[1];
66
+ items[0] = __half2half2(__low2half(i01));
67
+ items[1] = __half2half2(__high2half(i01));
68
+ items[2] = __half2half2(__low2half(i23));
69
+ items[3] = __half2half2(__high2half(i23));
70
+ }
71
};
72
65
-class MatrixView_half_rw
66
-{
67
-public:
68
- half* data;
69
- const int height;
70
- const int width;
71
-
72
- __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)
73
- : data(data), height(height), width(width)
74
- { }
75
-
76
- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
77
- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
78
- __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }
79
- __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }
80
- __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }
81
-
82
- __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)
83
- {
84
- half2 v01 = __halves2half2(v0, v1);
85
- half2 v23 = __halves2half2(v2, v3);
86
- half2* ptr = (half2*) item_ptr(row, column);
87
- ptr[0] = v01;
88
- ptr[1] = v23;
89
- }
73
+class MatrixView_half_rw {
74
+ public:
75
+ half* data;
76
+ const int height;
77
+ const int width;
78
+
79
+ __device__ __forceinline__ MatrixView_half_rw(half* data, const int height,
80
+ const int width)
81
+ : data(data), height(height), width(width) {}
82
+
83
+ __device__ __forceinline__ half item(int row, int column) const {
84
+ return data[row * width + column];
85
+ }
86
+ __device__ __forceinline__ half2 item_half2(int row, int column) const {
87
+ return ((half2*)data)[(row * width + column) / 2];
88
+ }
89
+ __device__ __forceinline__ half* item_ptr(int row, int column) {
90
+ return &data[row * width + column];
91
+ }
92
+ __device__ __forceinline__ void set(int row, int column, half value) {
93
+ data[row * width + column] = value;
94
+ }
95
+ __device__ __forceinline__ void set_half2(int row, int column, half2 value) {
96
+ ((half2*)data)[(row * width + column) / 2] = value;
97
+ }
98
+
99
+ __device__ __forceinline__ void set4(int row, int column, half v0, half v1,
100
+ half v2, half v3) {
101
+ half2 v01 = __halves2half2(v0, v1);
102
+ half2 v23 = __halves2half2(v2, v3);
103
+ half2* ptr = (half2*)item_ptr(row, column);
104
+ ptr[0] = v01;
105
+ ptr[1] = v23;
106
+ }
107
};
108
92
-class MatrixView_q4_row
93
-{
94
-public:
95
- const uint32_t* data;
96
- const int height;
97
- const int width;
98
-
99
- __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)
100
- : data(data), height(height), width(width)
101
- { }
102
-
103
- __device__ __forceinline__ int item(int row, int column) const
104
- {
105
- int shift = (column & 0x07) * 4;
106
- return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
107
- }
108
-
109
- __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
110
- {
111
- int shift = (column & 0x07) * 4;
112
- uint32_t d = data[row * width / 8 + column / 8] >> shift;
113
- items[0] = d & 0x0f;
114
- items[1] = (d >> 4) & 0x0f;
115
- }
116
-
117
- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
118
- {
119
- int shift = (column & 0x07) * 4;
120
- uint32_t d = data[row * width / 8 + column / 8] >> shift;
121
- items[0] = d & 0x0f;
122
- items[1] = (d >> 4) & 0x0f;
123
- items[2] = (d >> 8) & 0x0f;
124
- items[3] = (d >> 12) & 0x0f;
125
- }
109
+class MatrixView_q4_row {
110
+ public:
111
+ const uint32_t* data;
112
+ const int height;
113
+ const int width;
114
+
115
+ __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data,
116
+ const int height,
117
+ const int width)
118
+ : data(data), height(height), width(width) {}
119
+
120
+ __device__ __forceinline__ int item(int row, int column) const {
121
+ int shift = (column & 0x07) * 4;
122
+ return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
123
+ }
124
+
125
+ __device__ __forceinline__ void item2(int (&items)[2], int row,
126
+ int column) const {
127
+ int shift = (column & 0x07) * 4;
128
+ uint32_t d = data[row * width / 8 + column / 8] >> shift;
129
+ items[0] = d & 0x0f;
130
+ items[1] = (d >> 4) & 0x0f;
131
+ }
132
+
133
+ __device__ __forceinline__ void item4(int (&items)[4], int row,
134
+ int column) const {
135
+ int shift = (column & 0x07) * 4;
136
+ uint32_t d = data[row * width / 8 + column / 8] >> shift;
137
+ items[0] = d & 0x0f;
138
+ items[1] = (d >> 4) & 0x0f;
139
+ items[2] = (d >> 8) & 0x0f;
140
+ items[3] = (d >> 12) & 0x0f;
141
+ }
142
};
143
128
-class MatrixView_q4_column
129
-{
130
-public:
131
- const uint32_t* data;
132
- const int height;
133
- const int width;
134
-
135
- __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width)
136
- : data(data), height(height), width(width)
137
- { }
138
-
139
- __device__ __forceinline__ int item(int row, int column) const
140
- {
141
- int shift = (row & 0x07) * 4;
142
- return (data[row / 8 * width + column] >> shift) & 0x0f;
143
- }
144
-
145
- __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { return data[row / 8 * width + column]; }
146
- __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; }
144
+class MatrixView_q4_column {
145
+ public:
146
+ const uint32_t* data;
147
+ const int height;
148
+ const int width;
149
+
150
+ __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data,
151
+ const int height,
152
+ const int width)
153
+ : data(data), height(height), width(width) {}
154
+
155
+ __device__ __forceinline__ int item(int row, int column) const {
156
+ int shift = (row & 0x07) * 4;
157
+ return (data[row / 8 * width + column] >> shift) & 0x0f;
158
+ }
159
+
160
+ __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) {
161
+ return data[row / 8 * width + column];
162
+ }
163
+ __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row,
164
+ int column) {
165
+ return &data[row / 8 * width + column];
166
+ }
167
};
168
149
-class MatrixView_q2_row
150
-{
151
-public:
152
- const uint32_t* data;
153
- const int height;
154
- const int width;
155
-
156
- __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width)
157
- : data(data), height(height), width(width)
158
- { }
159
-
160
- __device__ __forceinline__ int item(int row, int column) const
161
- {
162
- int shift = (column & 0x0f) * 2;
163
- return (data[row * width / 16 + column / 16] >> shift) & 0x03;
164
- }
165
-
166
- __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
167
- {
168
- int shift = (column & 0x0f) * 2;
169
- uint32_t d = data[row * width / 16 + column / 16] >> shift;
170
- items[0] = d & 0x03;
171
- items[1] = (d >> 2) & 0x03;
172
- }
173
-
174
- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
175
- {
176
- int shift = (column & 0x0f) * 2;
177
- uint32_t d = data[row * width / 16 + column / 16] >> shift;
178
- items[0] = d & 0x03;
179
- items[1] = (d >> 2) & 0x03;
180
- items[2] = (d >> 4) & 0x03;
181
- items[3] = (d >> 6) & 0x03;
182
- }
169
+class MatrixView_q2_row {
170
+ public:
171
+ const uint32_t* data;
172
+ const int height;
173
+ const int width;
174
+
175
+ __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data,
176
+ const int height,
177
+ const int width)
178
+ : data(data), height(height), width(width) {}
179
+
180
+ __device__ __forceinline__ int item(int row, int column) const {
181
+ int shift = (column & 0x0f) * 2;
182
+ return (data[row * width / 16 + column / 16] >> shift) & 0x03;
183
+ }
184
+
185
+ __device__ __forceinline__ void item2(int (&items)[2], int row,
186
+ int column) const {
187
+ int shift = (column & 0x0f) * 2;
188
+ uint32_t d = data[row * width / 16 + column / 16] >> shift;
189
+ items[0] = d & 0x03;
190
+ items[1] = (d >> 2) & 0x03;
191
+ }
192
+
193
+ __device__ __forceinline__ void item4(int (&items)[4], int row,
194
+ int column) const {
195
+ int shift = (column & 0x0f) * 2;
196
+ uint32_t d = data[row * width / 16 + column / 16] >> shift;
197
+ items[0] = d & 0x03;
198
+ items[1] = (d >> 2) & 0x03;
199
+ items[2] = (d >> 4) & 0x03;
200
+ items[3] = (d >> 6) & 0x03;
201
+ }
202
};
203
185
-class MatrixView_q3_row
186
-{
187
-public:
188
- const uint32_t* data;
189
- const int height;
190
- const int width;
191
-
192
- __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width)
193
- : data(data), height(height), width(width)
194
- { }
195
-
196
- __device__ __forceinline__ int item(int row, int column) const
197
- {
198
- int z_w = column * 3 / 32;
199
- int z_mod = column & 0x1f;
200
-
201
- if (z_mod == 10) {
202
- return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
203
- } else if (z_mod == 21) {
204
- return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
205
- } else if (z_mod < 10) {
206
- return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
207
- } else if (z_mod < 21) {
208
- return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
209
- } else {
210
- return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
211
- }
204
+class MatrixView_q3_row {
205
+ public:
206
+ const uint32_t* data;
207
+ const int height;
208
+ const int width;
209
+
210
+ __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data,
211
+ const int height,
212
+ const int width)
213
+ : data(data), height(height), width(width) {}
214
+
215
+ __device__ __forceinline__ int item(int row, int column) const {
216
+ int z_w = column * 3 / 32;
217
+ int z_mod = column & 0x1f;
218
+
219
+ if (z_mod == 10) {
220
+ return (data[row * width * 3 / 32 + z_w] >> 30) |
221
+ ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4);
222
+ } else if (z_mod == 21) {
223
+ return (data[row * width * 3 / 32 + z_w] >> 31) |
224
+ ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6);
225
+ } else if (z_mod < 10) {
226
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07;
227
+ } else if (z_mod < 21) {
228
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07;
229
+ } else {
230
+ return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07;
231
}
213
-
214
- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
215
- {
216
- int shift = (column & 0x1f);
217
- uint32_t d;
218
- if (shift <= 4) {
219
- d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
220
- } else if (shift == 8) {
221
- d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
222
- } else if (shift <= 16) {
223
- d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
224
- } else if (shift == 20) {
225
- d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
226
- } else {
227
- d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
228
- }
229
- items[0] = d & 0x07;
230
- items[1] = (d >> 3) & 0x07;
231
- items[2] = (d >> 6) & 0x07;
232
- items[3] = (d >> 9) & 0x07;
232
+ }
233
+
234
+ __device__ __forceinline__ void item4(int (&items)[4], int row,
235
+ int column) const {
236
+ int shift = (column & 0x1f);
237
+ uint32_t d;
238
+ if (shift <= 4) {
239
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3);
240
+ } else if (shift == 8) {
241
+ d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) |
242
+ ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8);
243
+ } else if (shift <= 16) {
244
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32);
245
+ } else if (shift == 20) {
246
+ d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) |
247
+ ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4);
248
+ } else {
249
+ d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64);
250
}
251
+ items[0] = d & 0x07;
252
+ items[1] = (d >> 3) & 0x07;
253
+ items[2] = (d >> 6) & 0x07;
254
+ items[3] = (d >> 9) & 0x07;
255
+ }
256
};
257
236
-class MatrixView_q8_row
237
-{
238
-public:
239
- const uint32_t* data;
240
- const int height;
241
- const int width;
242
-
243
- __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width)
244
- : data(data), height(height), width(width)
245
- { }
246
-
247
- __device__ __forceinline__ int item(int row, int column) const
248
- {
249
- int shift = (column & 0x03) * 8;
250
- return (data[row * width / 4 + column / 4] >> shift) & 0xff;
251
- }
252
-
253
- __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
254
- {
255
- int shift = (column & 0x03) * 8;
256
- uint32_t d = data[row * width / 4 + column / 4] >> shift;
257
- items[0] = d & 0xff;
258
- items[1] = (d >> 8) & 0xff;
259
- }
260
-
261
- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
262
- {
263
- int shift = (column & 0x03) * 2;
264
- uint32_t d = data[row * width / 4 + column / 4] >> shift;
265
- items[0] = d & 0xff;
266
- items[1] = (d >> 8) & 0xff;
267
- items[2] = (d >> 16) & 0xff;
268
- items[3] = (d >> 24) & 0xff;
269
- }
258
+class MatrixView_q8_row {
259
+ public:
260
+ const uint32_t* data;
261
+ const int height;
262
+ const int width;
263
+
264
+ __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data,
265
+ const int height,
266
+ const int width)
267
+ : data(data), height(height), width(width) {}
268
+
269
+ __device__ __forceinline__ int item(int row, int column) const {
270
+ int shift = (column & 0x03) * 8;
271
+ return (data[row * width / 4 + column / 4] >> shift) & 0xff;
272
+ }
273
+
274
+ __device__ __forceinline__ void item2(int (&items)[2], int row,
275
+ int column) const {
276
+ int shift = (column & 0x03) * 8;
277
+ uint32_t d = data[row * width / 4 + column / 4] >> shift;
278
+ items[0] = d & 0xff;
279
+ items[1] = (d >> 8) & 0xff;
280
+ }
281
+
282
+ __device__ __forceinline__ void item4(int (&items)[4], int row,
283
+ int column) const {
284
+ int shift = (column & 0x03) * 2;
285
+ uint32_t d = data[row * width / 4 + column / 4] >> shift;
286
+ items[0] = d & 0xff;
287
+ items[1] = (d >> 8) & 0xff;
288
+ items[2] = (d >> 16) & 0xff;
289
+ items[3] = (d >> 24) & 0xff;
290
+ }
291
};
292
293
} // namespace gptq
csrc/quantization/gptq/q_gemm.cu
+1611
-1830
@@ -1,5 +1,6 @@
1
/*
2
-Adapted from https://github.com/turboderp/exllamav2 and https://github.com/qwopqwop200/GPTQ-for-LLaMa
2
+Adapted from https://github.com/turboderp/exllamav2 and
3
+https://github.com/qwopqwop200/GPTQ-for-LLaMa
4
*/
5
6
#include <cstdint>
@@ -32,2044 +33,1824 @@ namespace gptq {
33
#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
34
35
#if defined(USE_ROCM)
35
-#include <hipblas/hipblas.h>
36
-__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
37
- hipblasOperation_t transA,
38
- hipblasOperation_t transB,
39
- int m,
40
- int n,
41
- int k,
42
- const half* alpha,
43
- const half* AP,
44
- int lda,
45
- const half* BP,
46
- int ldb,
47
- const half* beta,
48
- half* CP,
49
- int ldc) {
50
- return hipblasHgemm(handle, transA, transB, m, n, k,
51
- reinterpret_cast<const hipblasHalf *>(alpha),
52
- reinterpret_cast<const hipblasHalf *>(AP), lda,
53
- reinterpret_cast<const hipblasHalf *>(BP), ldb,
54
- reinterpret_cast<const hipblasHalf *>(beta),
55
- reinterpret_cast<hipblasHalf *>(CP), ldc);
36
+ #include <hipblas/hipblas.h>
37
+__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(
38
+ hipblasHandle_t handle, hipblasOperation_t transA,
39
+ hipblasOperation_t transB, int m, int n, int k, const half* alpha,
40
+ const half* AP, int lda, const half* BP, int ldb, const half* beta,
41
+ half* CP, int ldc) {
42
+ return hipblasHgemm(handle, transA, transB, m, n, k,
43
+ reinterpret_cast<const hipblasHalf*>(alpha),
44
+ reinterpret_cast<const hipblasHalf*>(AP), lda,
45
+ reinterpret_cast<const hipblasHalf*>(BP), ldb,
46
+ reinterpret_cast<const hipblasHalf*>(beta),
47
+ reinterpret_cast<hipblasHalf*>(CP), ldc);
48
}
57
-#define hipblasHgemm __compat_hipblasHgemm
49
+ #define hipblasHgemm __compat_hipblasHgemm
50
59
-// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
60
-#define rocblas_operation_none HIPBLAS_OP_N
61
-#define rocblas_hgemm __compat_hipblasHgemm
51
+ // Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
52
+ #define rocblas_operation_none HIPBLAS_OP_N
53
+ #define rocblas_hgemm __compat_hipblasHgemm
54
#endif
55
64
-__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)
65
-{
66
- half2 result = {};
67
- const half2* a2_ptr = (const half2*)a_ptr;
68
- #pragma unroll
69
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
70
- return __hadd2(result, g_result);
56
+__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr,
57
+ const half2 g_result) {
58
+ half2 result = {};
59
+ const half2* a2_ptr = (const half2*)a_ptr;
60
+#pragma unroll
61
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
62
+ return __hadd2(result, g_result);
63
}
64
73
-__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
74
-{
75
- half2 result = {};
76
- const half2* a2_ptr = (const half2*)a_ptr;
77
- #pragma unroll
78
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
79
- return __half2float(__low2half(result)) + __half2float(__high2half(result));
65
+__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) {
66
+ half2 result = {};
67
+ const half2* a2_ptr = (const half2*)a_ptr;
68
+#pragma unroll
69
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
70
+ return __half2float(__low2half(result)) + __half2float(__high2half(result));
71
}
72
82
-__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)
83
-{
84
- half2 result = {};
85
- const half2* a2_ptr = (const half2*)a_ptr;
86
- #pragma unroll
87
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
88
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
73
+__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr,
74
+ const half2 g_result,
75
+ const half qs_h) {
76
+ half2 result = {};
77
+ const half2* a2_ptr = (const half2*)a_ptr;
78
+#pragma unroll
79
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
80
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
81
}
82
91
-__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)
92
-{
93
- half2 result = {};
94
- const half2* a2_ptr = (const half2*)a_ptr;
95
- #pragma unroll
96
- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
97
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
83
+__forceinline__ __device__ half2 dot22_16(half2 (&dq)[8], const half* a_ptr,
84
+ const half2 g_result,
85
+ const half qs_h) {
86
+ half2 result = {};
87
+ const half2* a2_ptr = (const half2*)a_ptr;
88
+#pragma unroll
89
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
90
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
91
}
92
100
-__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)
101
-{
102
- half2 result = {};
103
- const half2* a2_ptr = (const half2*)a_ptr;
104
- #pragma unroll
105
- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
106
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
93
+__forceinline__ __device__ half2 dot22_32(half2 (&dq)[16], const half* a_ptr,
94
+ const half2 g_result,
95
+ const half qs_h) {
96
+ half2 result = {};
97
+ const half2* a2_ptr = (const half2*)a_ptr;
98
+#pragma unroll
99
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
100
+ return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
101
}
102
109
-__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)
110
-{
111
- half2 result = {};
112
- const half2* a2_ptr = (const half2*)a_ptr;
113
- #pragma unroll
114
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
115
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
116
- return fma(result_f, qs_f, g_result);
103
+__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr,
104
+ const float g_result,
105
+ const float qs_f) {
106
+ half2 result = {};
107
+ const half2* a2_ptr = (const half2*)a_ptr;
108
+#pragma unroll
109
+ for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
110
+ float result_f =
111
+ __half2float(__low2half(result)) + __half2float(__high2half(result));
112
+ return fma(result_f, qs_f, g_result);
113
}
114
119
-__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)
120
-{
121
- half2 result = {};
122
- const half2* a2_ptr = (const half2*)a_ptr;
123
- #pragma unroll
124
- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
125
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
126
- return fma(result_f, qs_f, g_result);
115
+__forceinline__ __device__ float dot22_16_f(half2 (&dq)[8], const half* a_ptr,
116
+ const float g_result,
117
+ const float qs_f) {
118
+ half2 result = {};
119
+ const half2* a2_ptr = (const half2*)a_ptr;
120
+#pragma unroll
121
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
122
+ float result_f =
123
+ __half2float(__low2half(result)) + __half2float(__high2half(result));
124
+ return fma(result_f, qs_f, g_result);
125
}
126
129
-__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)
130
-{
131
- half2 result = {};
132
- const half2* a2_ptr = (const half2*)a_ptr;
133
- #pragma unroll
134
- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
135
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
136
- return fma(result_f, qs_f, g_result);
127
+__forceinline__ __device__ float dot22_32_f(half2 (&dq)[16], const half* a_ptr,
128
+ const float g_result,
129
+ const float qs_f) {
130
+ half2 result = {};
131
+ const half2* a2_ptr = (const half2*)a_ptr;
132
+#pragma unroll
133
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
134
+ float result_f =
135
+ __half2float(__low2half(result)) + __half2float(__high2half(result));
136
+ return fma(result_f, qs_f, g_result);
137
}
138
139
-__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h)
140
-{
141
- // Use FP32 accumulator to avoid potential overflow since unscaled weights are in the range -128..127
142
-
143
- float result = {};
144
- #pragma unroll
145
- for (int i = 0; i < 4; i++)
146
- {
147
- half2 w01 = dq[i];
148
- float w0 = __low2float(w01);
149
- float w1 = __high2float(w01);
150
- float x0 = __half2float(*a_ptr++);
151
- float x1 = __half2float(*a_ptr++);
152
- result = fma(w0, x0, result);
153
- result = fma(w1, x1, result);
154
- }
155
- float qs = __half2float(qs_h);
156
- result *= qs;
157
- half result_h = __float2half_rn(result);
158
- return __hadd(result_h, g_result);
139
+__forceinline__ __device__ half dot22_8_h(half2 (&dq)[4], const half* a_ptr,
140
+ const half g_result,
141
+ const half qs_h) {
142
+ // Use FP32 accumulator to avoid potential overflow since unscaled weights are
143
+ // in the range -128..127
144
+
145
+ float result = {};
146
+#pragma unroll
147
+ for (int i = 0; i < 4; i++) {
148
+ half2 w01 = dq[i];
149
+ float w0 = __low2float(w01);
150
+ float w1 = __high2float(w01);
151
+ float x0 = __half2float(*a_ptr++);
152
+ float x1 = __half2float(*a_ptr++);
153
+ result = fma(w0, x0, result);
154
+ result = fma(w1, x1, result);
155
+ }
156
+ float qs = __half2float(qs_h);
157
+ result *= qs;
158
+ half result_h = __float2half_rn(result);
159
+ return __hadd(result_h, g_result);
160
}
161
161
-__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h)
162
-{
163
- half2 result = {};
164
- const half2* a2_ptr = (const half2*)a_ptr;
165
- #pragma unroll
166
- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
167
- half result_h = __hadd(__low2half(result), __high2half(result));
168
- return __hfma(result_h, qs_h, g_result);
162
+__forceinline__ __device__ half dot22_16_h(half2 (&dq)[8], const half* a_ptr,
163
+ const half g_result,
164
+ const half qs_h) {
165
+ half2 result = {};
166
+ const half2* a2_ptr = (const half2*)a_ptr;
167
+#pragma unroll
168
+ for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
169
+ half result_h = __hadd(__low2half(result), __high2half(result));
170
+ return __hfma(result_h, qs_h, g_result);
171
}
172
171
-__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h)
172
-{
173
- half2 result = {};
174
- const half2* a2_ptr = (const half2*)a_ptr;
175
- #pragma unroll
176
- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
177
- half result_h = __hadd(__low2half(result), __high2half(result));
178
- return __hfma(result_h, qs_h, g_result);
173
+__forceinline__ __device__ half dot22_32_h(half2 (&dq)[16], const half* a_ptr,
174
+ const half g_result,
175
+ const half qs_h) {
176
+ half2 result = {};
177
+ const half2* a2_ptr = (const half2*)a_ptr;
178
+#pragma unroll
179
+ for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
180
+ half result_h = __hadd(__low2half(result), __high2half(result));
181
+ return __hfma(result_h, qs_h, g_result);
182
}
183
181
-
182
-typedef void (*fp_gemm_half_q_half_gptq_kernel)
183
-(
184
- const half*,
185
- const uint32_t*,
186
- const uint32_t*,
187
- const half*,
188
- half*,
189
- const int,
190
- const int,
191
- const int,
192
- const int,
193
- const int*
194
-);
195
-
184
+typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,
185
+ const uint32_t*, const half*,
186
+ half*, const int, const int,
187
+ const int, const int,
188
+ const int*);
189
190
template <bool first_block, int m_count>
198
-__global__ void gemm_half_q_half_gptq_4bit_kernel
199
-(
200
- const half* __restrict__ a,
201
- const uint32_t* __restrict__ b_q_weight,
191
+__global__ void gemm_half_q_half_gptq_4bit_kernel(
192
+ const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
193
const uint32_t* __restrict__ b_gptq_qzeros,
203
- const half* __restrict__ b_gptq_scales,
204
- half* __restrict__ c,
205
- const int size_m,
206
- const int size_n,
207
- const int size_k,
208
- const int groups,
209
- const int* __restrict__ b_q_perm
210
-)
211
-{
212
- MatrixView_half a_(a, size_m, size_k);
213
- MatrixView_half_rw c_(c, size_m, size_n);
214
- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
215
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
216
-
217
- int t = threadIdx.x;
218
-
219
- // Block
220
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
221
- int offset_m = blockIdx.y * m_count;
222
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
223
-
224
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
225
- int end_m = min(offset_m + m_count, size_m);
226
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
227
-
228
- int n = offset_n + t * 4;
229
-
230
- // Preload block_a
231
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
232
-
233
- if (offset_k + t < end_k)
234
- {
235
- for (int m = 0; m < m_count; ++m)
236
- {
237
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
238
- half* block_a_ptr = block_a[m];
239
-
240
- half a0;
241
- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
242
- else a0 = a_ptr[offset_k + t];
243
- block_a_ptr[t] = a0;
244
- }
194
+ const half* __restrict__ b_gptq_scales, half* __restrict__ c,
195
+ const int size_m, const int size_n, const int size_k, const int groups,
196
+ const int* __restrict__ b_q_perm) {
197
+ MatrixView_half a_(a, size_m, size_k);
198
+ MatrixView_half_rw c_(c, size_m, size_n);
199
+ MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
200
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
201
+
202
+ int t = threadIdx.x;
203
+
204
+ // Block
205
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
206
+ int offset_m = blockIdx.y * m_count;
207
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
208
+
209
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
210
+ int end_m = min(offset_m + m_count, size_m);
211
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
212
+
213
+ int n = offset_n + t * 4;
214
+
215
+ // Preload block_a
216
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
217
+
218
+ if (offset_k + t < end_k) {
219
+ for (int m = 0; m < m_count; ++m) {
220
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
221
+ half* block_a_ptr = block_a[m];
222
+
223
+ half a0;
224
+ if (b_q_perm)
225
+ a0 = a_ptr[b_q_perm[offset_k + t]];
226
+ else
227
+ a0 = a_ptr[offset_k + t];
228
+ block_a_ptr[t] = a0;
229
}
230
+ }
231
247
- // Zero output
248
- if (n >= size_n) return;
232
+ // Zero output
233
+ if (n >= size_n) return;
234
250
- if (blockIdx.z == 0)
251
- {
252
- for (int m = 0; m < m_count; m++)
253
- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
235
+ if (blockIdx.z == 0) {
236
+ for (int m = 0; m < m_count; m++)
237
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
238
+ }
239
+
240
+ __syncthreads();
241
+
242
+ // Find initial group
243
+ int groupsize = size_k / groups;
244
+ int group = offset_k / groupsize;
245
+ int nextgroup = offset_k + groupsize;
246
+
247
+ // a, b offset
248
+ int qk = offset_k / (32 / 4);
249
+
250
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
251
+ const half* a_ptr = &block_a[0][0];
252
+ int a_stride = BLOCK_KN_SIZE;
253
+
254
+ // Initial group
255
+ int zeros[4];
256
+ float scales[4];
257
+ half2 z1z16[4][2];
258
+ half2 y1y16[4][2];
259
+ b_gptq_qzeros_.item4(zeros, group, n);
260
+ b_gptq_scales_.item4_f(scales, group, n);
261
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
262
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
263
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
264
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
265
+
266
+ // Column result
267
+ float block_c[m_count][4] = {};
268
+
269
+ // Dequantize and multiply
270
+ int k = offset_k;
271
+ while (k < end_k) {
272
+ if (k == nextgroup) {
273
+ group++;
274
+ nextgroup += groupsize;
275
+ b_gptq_qzeros_.item4(zeros, group, n);
276
+ b_gptq_scales_.item4_f(scales, group, n);
277
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
278
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
279
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
280
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
281
}
282
256
- __syncthreads();
257
-
258
- // Find initial group
259
- int groupsize = size_k / groups;
260
- int group = offset_k / groupsize;
261
- int nextgroup = offset_k + groupsize;
262
-
263
- // a, b offset
264
- int qk = offset_k / (32 / 4);
265
-
266
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
267
- const half* a_ptr = &block_a[0][0];
268
- int a_stride = BLOCK_KN_SIZE;
269
-
270
- // Initial group
271
- int zeros[4];
272
- float scales[4];
273
- half2 z1z16[4][2];
274
- half2 y1y16[4][2];
275
- b_gptq_qzeros_.item4(zeros, group, n);
276
- b_gptq_scales_.item4_f(scales, group, n);
277
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
278
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
279
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
280
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
281
-
282
- // Column result
283
- float block_c[m_count][4] = {};
284
-
285
- // Dequantize and multiply
286
- int k = offset_k;
287
- while (k < end_k)
288
- {
289
- if (k == nextgroup)
290
- {
291
- group++;
292
- nextgroup += groupsize;
293
- b_gptq_qzeros_.item4(zeros, group, n);
294
- b_gptq_scales_.item4_f(scales, group, n);
295
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
296
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
297
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
298
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
299
- }
300
-
301
- #pragma unroll
302
- for (int j = 0; j < 4; j++)
303
- {
304
- const int4* b_ptr4 = (int4*) b_ptr;
305
- int4 load_int4 = *b_ptr4;
306
-
307
- half2 dq[4][4];
308
- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
309
- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
310
- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
311
- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
312
-
313
- #pragma unroll
314
- for (int m = 0; m < m_count; m++)
315
- {
316
- block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);
317
- block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);
318
- block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);
319
- block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);
320
- }
321
-
322
- b_ptr += size_n;
323
- a_ptr += 8;
324
- }
325
-
326
- k += 32;
283
+#pragma unroll
284
+ for (int j = 0; j < 4; j++) {
285
+ const int4* b_ptr4 = (int4*)b_ptr;
286
+ int4 load_int4 = *b_ptr4;
287
+
288
+ half2 dq[4][4];
289
+ dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n,
290
+ false);
291
+ dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n,
292
+ false);
293
+ dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n,
294
+ false);
295
+ dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n,
296
+ false);
297
+
298
+#pragma unroll
299
+ for (int m = 0; m < m_count; m++) {
300
+ block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0],
301
+ block_c[m][0]);
302
+ block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1],
303
+ block_c[m][1]);
304
+ block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2],
305
+ block_c[m][2]);
306
+ block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3],
307
+ block_c[m][3]);
308
+ }
309
+
310
+ b_ptr += size_n;
311
+ a_ptr += 8;
312
}
313
329
- for (int m = 0; m < m_count; m++)
330
- {
331
- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
332
- half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
333
- half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
334
- atomicAdd(out , result01);
335
- atomicAdd(out + 1, result23);
336
- }
314
+ k += 32;
315
+ }
316
+
317
+ for (int m = 0; m < m_count; m++) {
318
+ half2* out = (half2*)c_.item_ptr(offset_m + m, n);
319
+ half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]),
320
+ __float2half_rn(block_c[m][1]));
321
+ half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]),
322
+ __float2half_rn(block_c[m][3]));
323
+ atomicAdd(out, result01);
324
+ atomicAdd(out + 1, result23);
325
+ }
326
}
327
328
template <bool first_block, int m_count>
340
-__global__ void gemm_half_q_half_gptq_2bit_kernel
341
-(
342
- const half* __restrict__ a,
343
- const uint32_t* __restrict__ b_q_weight,
329
+__global__ void gemm_half_q_half_gptq_2bit_kernel(
330
+ const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
331
const uint32_t* __restrict__ b_gptq_qzeros,
345
- const half* __restrict__ b_gptq_scales,
346
- half* __restrict__ c,
347
- const int size_m,
348
- const int size_n,
349
- const int size_k,
350
- const int groups,
351
- const int* __restrict__ b_q_perm
352
-)
353
-{
354
- MatrixView_half a_(a, size_m, size_k);
355
- MatrixView_half_rw c_(c, size_m, size_n);
356
- MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
357
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
358
-
359
- int t = threadIdx.x;
360
-
361
- // Block
362
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
363
- int offset_m = blockIdx.y * m_count;
364
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
365
-
366
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
367
- int end_m = min(offset_m + m_count, size_m);
368
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
369
-
370
- int n = offset_n + t * 4;
371
-
372
- // Preload block_a
373
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
374
-
375
- if (offset_k + t < end_k)
376
- {
377
- for (int m = 0; m < m_count; ++m)
378
- {
379
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
380
- half* block_a_ptr = block_a[m];
381
-
382
- half a0;
383
- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
384
- else a0 = a_ptr[offset_k + t];
385
- block_a_ptr[t] = a0;
386
- }
332
+ const half* __restrict__ b_gptq_scales, half* __restrict__ c,
333
+ const int size_m, const int size_n, const int size_k, const int groups,
334
+ const int* __restrict__ b_q_perm) {
335
+ MatrixView_half a_(a, size_m, size_k);
336
+ MatrixView_half_rw c_(c, size_m, size_n);
337
+ MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
338
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
339
+
340
+ int t = threadIdx.x;
341
+
342
+ // Block
343
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
344
+ int offset_m = blockIdx.y * m_count;
345
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
346
+
347
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
348
+ int end_m = min(offset_m + m_count, size_m);
349
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
350
+
351
+ int n = offset_n + t * 4;
352
+
353
+ // Preload block_a
354
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
355
+
356
+ if (offset_k + t < end_k) {
357
+ for (int m = 0; m < m_count; ++m) {
358
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
359
+ half* block_a_ptr = block_a[m];
360
+
361
+ half a0;
362
+ if (b_q_perm)
363
+ a0 = a_ptr[b_q_perm[offset_k + t]];
364
+ else
365
+ a0 = a_ptr[offset_k + t];
366
+ block_a_ptr[t] = a0;
367
}
368
+ }
369
389
- // Zero output
390
- if (n >= size_n) return;
370
+ // Zero output
371
+ if (n >= size_n) return;
372
392
- if (blockIdx.z == 0)
393
- {
394
- for (int m = 0; m < m_count; m++)
395
- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
373
+ if (blockIdx.z == 0) {
374
+ for (int m = 0; m < m_count; m++)
375
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
376
+ }
377
+
378
+ __syncthreads();
379
+
380
+ // Find initial group
381
+ int groupsize = size_k / groups;
382
+ int group = offset_k / groupsize;
383
+ int nextgroup = offset_k + groupsize;
384
+
385
+ // a, b offset
386
+ int qk = offset_k / (32 / 2);
387
+
388
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
389
+ const half* a_ptr = &block_a[0][0];
390
+ int a_stride = BLOCK_KN_SIZE;
391
+
392
+ // Initial group
393
+ int zeros[4];
394
+ half scales[4];
395
+ b_gptq_qzeros_.item4(zeros, group, n);
396
+ b_gptq_scales_.item4(scales, group, n);
397
+ // Column result
398
+ half block_c[m_count][4] = {};
399
+
400
+ // Dequantize and multiply
401
+ int k = offset_k;
402
+ while (k < end_k) {
403
+ if (k == nextgroup) {
404
+ group++;
405
+ nextgroup += groupsize;
406
+ b_gptq_qzeros_.item4(zeros, group, n);
407
+ b_gptq_scales_.item4(scales, group, n);
408
}
409
398
- __syncthreads();
399
-
400
- // Find initial group
401
- int groupsize = size_k / groups;
402
- int group = offset_k / groupsize;
403
- int nextgroup = offset_k + groupsize;
404
-
405
- // a, b offset
406
- int qk = offset_k / (32 / 2);
407
-
408
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
409
- const half* a_ptr = &block_a[0][0];
410
- int a_stride = BLOCK_KN_SIZE;
411
-
412
- // Initial group
413
- int zeros[4];
414
- half scales[4];
415
- b_gptq_qzeros_.item4(zeros, group, n);
416
- b_gptq_scales_.item4(scales, group, n);
417
- // Column result
418
- half block_c[m_count][4] = {};
419
-
420
- // Dequantize and multiply
421
- int k = offset_k;
422
- while (k < end_k)
423
- {
424
- if (k == nextgroup)
425
- {
426
- group++;
427
- nextgroup += groupsize;
428
- b_gptq_qzeros_.item4(zeros, group, n);
429
- b_gptq_scales_.item4(scales, group, n);
430
- }
431
-
432
- #pragma unroll
433
- for (int j = 0; j < 1; j++)
434
- {
435
- const int4* b_ptr4 = (int4*) b_ptr;
436
- int4 load_int4 = *b_ptr4;
437
-
438
- half2 dq[4][8];
439
- dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
440
- dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
441
- dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
442
- dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
443
-
444
- #pragma unroll
445
- for (int m = 0; m < m_count; m++)
446
- {
447
- block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
448
- block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
449
- block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
450
- block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
451
- }
452
-
453
- b_ptr += size_n;
454
- a_ptr += 16;
455
- }
456
-
457
- k += 16;
410
+#pragma unroll
411
+ for (int j = 0; j < 1; j++) {
412
+ const int4* b_ptr4 = (int4*)b_ptr;
413
+ int4 load_int4 = *b_ptr4;
414
+
415
+ half2 dq[4][8];
416
+ dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
417
+ dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
418
+ dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
419
+ dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
420
+
421
+#pragma unroll
422
+ for (int m = 0; m < m_count; m++) {
423
+ block_c[m][0] =
424
+ dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
425
+ block_c[m][1] =
426
+ dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
427
+ block_c[m][2] =
428
+ dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
429
+ block_c[m][3] =
430
+ dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
431
+ }
432
+
433
+ b_ptr += size_n;
434
+ a_ptr += 16;
435
}
436
460
- for (int m = 0; m < m_count; m++)
461
- {
462
- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
463
- half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
464
- half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
465
- atomicAdd(out , result01);
466
- atomicAdd(out + 1, result23);
467
- }
437
+ k += 16;
438
+ }
439
+
440
+ for (int m = 0; m < m_count; m++) {
441
+ half2* out = (half2*)c_.item_ptr(offset_m + m, n);
442
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
443
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
444
+ atomicAdd(out, result01);
445
+ atomicAdd(out + 1, result23);
446
+ }
447
}
448
449
template <bool first_block, int m_count>
471
-__global__ void gemm_half_q_half_gptq_3bit_kernel
472
-(
473
- const half* __restrict__ a,
474
- const uint32_t* __restrict__ b_q_weight,
450
+__global__ void gemm_half_q_half_gptq_3bit_kernel(
451
+ const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
452
const uint32_t* __restrict__ b_gptq_qzeros,
476
- const half* __restrict__ b_gptq_scales,
477
- half* __restrict__ c,
478
- const int size_m,
479
- const int size_n,
480
- const int size_k,
481
- const int groups,
482
- const int* __restrict__ b_q_perm
483
-)
484
-{
485
- MatrixView_half a_(a, size_m, size_k);
486
- MatrixView_half_rw c_(c, size_m, size_n);
487
- MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
488
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
489
-
490
- int t = threadIdx.x;
491
-
492
- // Block
493
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
494
- int offset_m = blockIdx.y * m_count;
495
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
496
-
497
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
498
- int end_m = min(offset_m + m_count, size_m);
499
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
500
-
501
- int n = offset_n + t * 4;
502
-
503
- // Preload block_a
504
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
505
-
506
- if (offset_k + t < end_k)
507
- {
508
- for (int m = 0; m < m_count; ++m)
509
- {
510
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
511
- half* block_a_ptr = block_a[m];
512
-
513
- half a0;
514
- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
515
- else a0 = a_ptr[offset_k + t];
516
- block_a_ptr[t] = a0;
517
- }
453
+ const half* __restrict__ b_gptq_scales, half* __restrict__ c,
454
+ const int size_m, const int size_n, const int size_k, const int groups,
455
+ const int* __restrict__ b_q_perm) {
456
+ MatrixView_half a_(a, size_m, size_k);
457
+ MatrixView_half_rw c_(c, size_m, size_n);
458
+ MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
459
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
460
+
461
+ int t = threadIdx.x;
462
+
463
+ // Block
464
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
465
+ int offset_m = blockIdx.y * m_count;
466
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
467
+
468
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
469
+ int end_m = min(offset_m + m_count, size_m);
470
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
471
+
472
+ int n = offset_n + t * 4;
473
+
474
+ // Preload block_a
475
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
476
+
477
+ if (offset_k + t < end_k) {
478
+ for (int m = 0; m < m_count; ++m) {
479
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
480
+ half* block_a_ptr = block_a[m];
481
+
482
+ half a0;
483
+ if (b_q_perm)
484
+ a0 = a_ptr[b_q_perm[offset_k + t]];
485
+ else
486
+ a0 = a_ptr[offset_k + t];
487
+ block_a_ptr[t] = a0;
488
}
489
+ }
490
520
- // Zero output
521
- if (n >= size_n) return;
491
+ // Zero output
492
+ if (n >= size_n) return;
493
523
- if (blockIdx.z == 0)
524
- {
525
- for (int m = 0; m < m_count; m++)
526
- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
494
+ if (blockIdx.z == 0) {
495
+ for (int m = 0; m < m_count; m++)
496
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
497
+ }
498
+
499
+ __syncthreads();
500
+
501
+ // Find initial group
502
+ int groupsize = size_k / groups;
503
+ int group = offset_k / groupsize;
504
+ int nextgroup = offset_k + groupsize;
505
+
506
+ // a, b offset
507
+ int qk = offset_k / 32 * 3;
508
+
509
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
510
+ const half* a_ptr = &block_a[0][0];
511
+ int a_stride = BLOCK_KN_SIZE;
512
+
513
+ // Initial group
514
+ int zeros[4];
515
+ half scales[4];
516
+ b_gptq_qzeros_.item4(zeros, group, n);
517
+ b_gptq_scales_.item4(scales, group, n);
518
+ // Column result
519
+ half block_c[m_count][4] = {};
520
+
521
+ // Dequantize and multiply
522
+ int k = offset_k;
523
+ while (k < end_k) {
524
+ if (k == nextgroup) {
525
+ group++;
526
+ nextgroup += groupsize;
527
+ b_gptq_qzeros_.item4(zeros, group, n);
528
+ b_gptq_scales_.item4(scales, group, n);
529
}
530
529
- __syncthreads();
530
-
531
- // Find initial group
532
- int groupsize = size_k / groups;
533
- int group = offset_k / groupsize;
534
- int nextgroup = offset_k + groupsize;
535
-
536
- // a, b offset
537
- int qk = offset_k / 32 * 3;
538
-
539
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
540
- const half* a_ptr = &block_a[0][0];
541
- int a_stride = BLOCK_KN_SIZE;
542
-
543
- // Initial group
544
- int zeros[4];
545
- half scales[4];
546
- b_gptq_qzeros_.item4(zeros, group, n);
547
- b_gptq_scales_.item4(scales, group, n);
548
- // Column result
549
- half block_c[m_count][4] = {};
550
-
551
- // Dequantize and multiply
552
- int k = offset_k;
553
- while (k < end_k)
554
- {
555
- if (k == nextgroup)
556
- {
557
- group++;
558
- nextgroup += groupsize;
559
- b_gptq_qzeros_.item4(zeros, group, n);
560
- b_gptq_scales_.item4(scales, group, n);
561
- }
562
-
563
- #pragma unroll
564
- for (int j = 0; j < 1; j++)
565
- {
566
- int4 load_int4[3];
567
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
568
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
569
- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
570
-
571
- half2 dq[4][16];
572
- dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
573
- dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
574
- dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
575
- dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
576
-
577
- #pragma unroll
578
- for (int m = 0; m < m_count; m++)
579
- {
580
- block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
581
- block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
582
- block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
583
- block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
584
- }
585
- a_ptr += 32;
586
- }
587
-
588
- k += 32;
531
+#pragma unroll
532
+ for (int j = 0; j < 1; j++) {
533
+ int4 load_int4[3];
534
+ load_int4[0] = *((int4*)b_ptr);
535
+ b_ptr += size_n;
536
+ load_int4[1] = *((int4*)b_ptr);
537
+ b_ptr += size_n;
538
+ load_int4[2] = *((int4*)b_ptr);
539
+ b_ptr += size_n;
540
+
541
+ half2 dq[4][16];
542
+ dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],
543
+ size_n, zeros[0] + 1);
544
+ dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],
545
+ size_n, zeros[1] + 1);
546
+ dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],
547
+ size_n, zeros[2] + 1);
548
+ dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
549
+ size_n, zeros[3] + 1);
550
+
551
+#pragma unroll
552
+ for (int m = 0; m < m_count; m++) {
553
+ block_c[m][0] =
554
+ dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
555
+ block_c[m][1] =
556
+ dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
557
+ block_c[m][2] =
558
+ dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
559
+ block_c[m][3] =
560
+ dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
561
+ }
562
+ a_ptr += 32;
563
}
564
591
- for (int m = 0; m < m_count; m++)
592
- {
593
- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
594
- half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
595
- half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
596
- atomicAdd(out , result01);
597
- atomicAdd(out + 1, result23);
598
- }
565
+ k += 32;
566
+ }
567
+
568
+ for (int m = 0; m < m_count; m++) {
569
+ half2* out = (half2*)c_.item_ptr(offset_m + m, n);
570
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
571
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
572
+ atomicAdd(out, result01);
573
+ atomicAdd(out + 1, result23);
574
+ }
575
}
576
577
template <bool first_block, int m_count>
602
-__global__ void gemm_half_q_half_gptq_8bit_kernel
603
-(
604
- const half* __restrict__ a,
605
- const uint32_t* __restrict__ b_q_weight,
578
+__global__ void gemm_half_q_half_gptq_8bit_kernel(
579
+ const half* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
580
const uint32_t* __restrict__ b_gptq_qzeros,
607
- const half* __restrict__ b_gptq_scales,
608
- half* __restrict__ c,
609
- const int size_m,
610
- const int size_n,
611
- const int size_k,
612
- const int groups,
613
- const int* __restrict__ b_q_perm
614
-)
615
-{
616
- MatrixView_half a_(a, size_m, size_k);
617
- MatrixView_half_rw c_(c, size_m, size_n);
618
- MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
619
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
620
-
621
- int t = threadIdx.x;
622
-
623
- // Block
624
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
625
- int offset_m = blockIdx.y * m_count;
626
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
627
-
628
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
629
- int end_m = min(offset_m + m_count, size_m);
630
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
631
-
632
- int n = offset_n + t * 4;
633
-
634
- // Preload block_a
635
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
636
-
637
- if (offset_k + t < end_k)
638
- {
639
- for (int m = 0; m < m_count; ++m)
640
- {
641
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
642
- half* block_a_ptr = block_a[m];
643
-
644
- half a0;
645
- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
646
- else a0 = a_ptr[offset_k + t];
647
- block_a_ptr[t] = a0;
648
- }
649
- }
650
-
651
- // Zero output
652
- if (n >= size_n) return;
653
-
654
- if (blockIdx.z == 0)
655
- {
656
- for (int m = 0; m < m_count; m++)
657
- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
581
+ const half* __restrict__ b_gptq_scales, half* __restrict__ c,
582
+ const int size_m, const int size_n, const int size_k, const int groups,
583
+ const int* __restrict__ b_q_perm) {
584
+ MatrixView_half a_(a, size_m, size_k);
585
+ MatrixView_half_rw c_(c, size_m, size_n);
586
+ MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
587
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
588
+
589
+ int t = threadIdx.x;
590
+
591
+ // Block
592
+ int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
593
+ int offset_m = blockIdx.y * m_count;
594
+ int offset_k = blockIdx.z * BLOCK_KN_SIZE;
595
+
596
+ int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
597
+ int end_m = min(offset_m + m_count, size_m);
598
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
599
+
600
+ int n = offset_n + t * 4;
601
+
602
+ // Preload block_a
603
+ __shared__ half block_a[m_count][BLOCK_KN_SIZE];
604
+
605
+ if (offset_k + t < end_k) {
606
+ for (int m = 0; m < m_count; ++m) {
607
+ const half* a_ptr = a_.item_ptr(offset_m + m, 0);
608
+ half* block_a_ptr = block_a[m];
609
+
610
+ half a0;
611
+ if (b_q_perm)
612
+ a0 = a_ptr[b_q_perm[offset_k + t]];
613
+ else
614
+ a0 = a_ptr[offset_k + t];
615
+ block_a_ptr[t] = a0;
616
}
617
+ }
618
660
- __syncthreads();
661
-
662
- // Find initial group
663
- int groupsize = size_k / groups;
664
- int group = offset_k / groupsize;
665
- int nextgroup = offset_k + groupsize;
666
-
667
- // a, b offset
668
- int qk = offset_k / (32 / 8);
669
-
670
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
671
- const half* a_ptr = &block_a[0][0];
672
- int a_stride = BLOCK_KN_SIZE;
673
-
674
- // Initial group
675
- int zeros[4];
676
- half scales[4];
677
- b_gptq_qzeros_.item4(zeros, group, n);
678
- b_gptq_scales_.item4(scales, group, n);
679
- // Column result
680
- half block_c[m_count][4] = {};
681
-
682
- // Dequantize and multiply
683
- int k = offset_k;
684
- while (k < end_k)
685
- {
686
- if (k == nextgroup)
687
- {
688
- group++;
689
- nextgroup += groupsize;
690
- b_gptq_qzeros_.item4(zeros, group, n);
691
- b_gptq_scales_.item4(scales, group, n);
692
- }
619
+ // Zero output
620
+ if (n >= size_n) return;
621
694
- #pragma unroll
695
- for (int j = 0; j < 4; j++)
696
- {
697
- int4 load_int4[2];
698
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
699
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
700
-
701
- half2 dq[4][4];
702
- dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
703
- dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
704
- dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
705
- dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
706
-
707
- for (int m = 0; m < m_count; m++)
708
- {
709
- block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
710
- block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
711
- block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
712
- block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
713
- }
714
- a_ptr += 8;
715
- }
716
- k += 32;
622
+ if (blockIdx.z == 0) {
623
+ for (int m = 0; m < m_count; m++)
624
+ *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
625
+ }
626
+
627
+ __syncthreads();
628
+
629
+ // Find initial group
630
+ int groupsize = size_k / groups;
631
+ int group = offset_k / groupsize;
632
+ int nextgroup = offset_k + groupsize;
633
+
634
+ // a, b offset
635
+ int qk = offset_k / (32 / 8);
636
+
637
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
638
+ const half* a_ptr = &block_a[0][0];
639
+ int a_stride = BLOCK_KN_SIZE;
640
+
641
+ // Initial group
642
+ int zeros[4];
643
+ half scales[4];
644
+ b_gptq_qzeros_.item4(zeros, group, n);
645
+ b_gptq_scales_.item4(scales, group, n);
646
+ // Column result
647
+ half block_c[m_count][4] = {};
648
+
649
+ // Dequantize and multiply
650
+ int k = offset_k;
651
+ while (k < end_k) {
652
+ if (k == nextgroup) {
653
+ group++;
654
+ nextgroup += groupsize;
655
+ b_gptq_qzeros_.item4(zeros, group, n);
656
+ b_gptq_scales_.item4(scales, group, n);
657
}
658
719
- for (int m = 0; m < m_count; m++)
720
- {
721
- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
722
- half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
723
- half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
724
- atomicAdd(out , result01);
725
- atomicAdd(out + 1, result23);
659
+#pragma unroll
660
+ for (int j = 0; j < 4; j++) {
661
+ int4 load_int4[2];
662
+ load_int4[0] = *((int4*)b_ptr);
663
+ b_ptr += size_n;
664
+ load_int4[1] = *((int4*)b_ptr);
665
+ b_ptr += size_n;
666
+
667
+ half2 dq[4][4];
668
+ dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,
669
+ zeros[0] + 1);
670
+ dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,
671
+ zeros[1] + 1);
672
+ dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,
673
+ zeros[2] + 1);
674
+ dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
675
+ zeros[3] + 1);
676
+
677
+ for (int m = 0; m < m_count; m++) {
678
+ block_c[m][0] =
679
+ dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]);
680
+ block_c[m][1] =
681
+ dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]);
682
+ block_c[m][2] =
683
+ dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]);
684
+ block_c[m][3] =
685
+ dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]);
686
+ }
687
+ a_ptr += 8;
688
}
689
+ k += 32;
690
+ }
691
+
692
+ for (int m = 0; m < m_count; m++) {
693
+ half2* out = (half2*)c_.item_ptr(offset_m + m, n);
694
+ half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]);
695
+ half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]);
696
+ atomicAdd(out, result01);
697
+ atomicAdd(out + 1, result23);
698
+ }
699
}
700
701
fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
730
- bool first_block, const int m_count, const int bit)
731
-{
732
- #define SELECT_KERNEL(M_COUNT) \
733
- if (m_count == M_COUNT) { \
734
- if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
735
- if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
736
- if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
737
- if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
738
- }
739
- #if BLOCK_M_SIZE_MAX >= 1
740
- SELECT_KERNEL(1);
741
- #endif
742
- #if BLOCK_M_SIZE_MAX >= 2
743
- SELECT_KERNEL(2);
744
- #endif
745
- #if BLOCK_M_SIZE_MAX >= 3
746
- SELECT_KERNEL(3);
747
- #endif
748
- #if BLOCK_M_SIZE_MAX >= 4
749
- SELECT_KERNEL(4);
750
- #endif
751
- #if BLOCK_M_SIZE_MAX >= 5
752
- SELECT_KERNEL(5);
753
- #endif
754
- #if BLOCK_M_SIZE_MAX >= 6
755
- SELECT_KERNEL(6);
756
- #endif
757
- #if BLOCK_M_SIZE_MAX >= 7
758
- SELECT_KERNEL(7);
759
- #endif
760
- #if BLOCK_M_SIZE_MAX >= 8
761
- SELECT_KERNEL(8);
762
- #endif
763
- return NULL;
702
+ bool first_block, const int m_count, const int bit) {
703
+#define SELECT_KERNEL(M_COUNT) \
704
+ if (m_count == M_COUNT) { \
705
+ if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel<true, M_COUNT>; \
706
+ if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel<true, M_COUNT>; \
707
+ if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel<true, M_COUNT>; \
708
+ if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel<true, M_COUNT>; \
709
+ }
710
+#if BLOCK_M_SIZE_MAX >= 1
711
+ SELECT_KERNEL(1);
712
+#endif
713
+#if BLOCK_M_SIZE_MAX >= 2
714
+ SELECT_KERNEL(2);
715
+#endif
716
+#if BLOCK_M_SIZE_MAX >= 3
717
+ SELECT_KERNEL(3);
718
+#endif
719
+#if BLOCK_M_SIZE_MAX >= 4
720
+ SELECT_KERNEL(4);
721
+#endif
722
+#if BLOCK_M_SIZE_MAX >= 5
723
+ SELECT_KERNEL(5);
724
+#endif
725
+#if BLOCK_M_SIZE_MAX >= 6
726
+ SELECT_KERNEL(6);
727
+#endif
728
+#if BLOCK_M_SIZE_MAX >= 7
729
+ SELECT_KERNEL(7);
730
+#endif
731
+#if BLOCK_M_SIZE_MAX >= 8
732
+ SELECT_KERNEL(8);
733
+#endif
734
+ return NULL;
735
}
736
737
+void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
738
+ const uint32_t* b_gptq_qzeros,
739
+ const half* b_gptq_scales, const int* b_q_perm,
740
+ half* c, int size_m, int size_n, int size_k,
741
+ int m_count, int groups, int bit) {
742
+ dim3 blockDim, gridDim;
743
+ blockDim.x = BLOCK_KN_SIZE;
744
+ blockDim.y = 1;
745
+ blockDim.z = 1;
746
+ gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
747
+ gridDim.y = DIVIDE(size_m, m_count);
748
+ gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
749
+
750
+ fp_gemm_half_q_half_gptq_kernel kernel =
751
+ pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
752
+
753
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
754
+ kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,
755
+ b_gptq_scales, c, size_m, size_n,
756
+ size_k, groups, b_q_perm);
757
+}
758
767
-void gemm_half_q_half_cuda_part
768
-(
769
- const half* a,
770
- const uint32_t* b_q_weight,
771
- const uint32_t* b_gptq_qzeros,
772
- const half* b_gptq_scales,
773
- const int* b_q_perm,
774
- half* c,
775
- int size_m,
776
- int size_n,
777
- int size_k,
778
- int m_count,
779
- int groups,
780
- int bit
781
-)
782
-{
783
- dim3 blockDim, gridDim;
784
- blockDim.x = BLOCK_KN_SIZE;
785
- blockDim.y = 1;
786
- blockDim.z = 1;
787
- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
788
- gridDim.y = DIVIDE(size_m, m_count);
789
- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
759
+__global__ void reconstruct_exllama_8bit_kernel(
760
+ const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
761
+ const uint32_t* __restrict__ b_gptq_qzeros,
762
+ const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
763
+ const int groups, half* __restrict__ b) {
764
+ MatrixView_half_rw b_(b, size_k, size_n);
765
+ MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
766
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
767
791
- fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
768
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
769
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
770
793
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
794
- kernel<<<gridDim, blockDim, 0, stream>>>
795
- (
796
- a,
797
- b_q_weight,
798
- b_gptq_qzeros,
799
- b_gptq_scales,
800
- c,
801
- size_m,
802
- size_n,
803
- size_k,
804
- groups,
805
- b_q_perm
806
- );
807
-}
771
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
772
773
+ // Preload remapping table
774
+ __shared__ int perm[BLOCK_KN_SIZE];
775
+ int t = threadIdx.x;
776
810
-__global__ void reconstruct_exllama_8bit_kernel
811
-(
812
- const uint32_t* __restrict__ b_q_weight,
813
- const int* __restrict__ b_q_perm,
814
- const uint32_t* __restrict__ b_gptq_qzeros,
815
- const half* __restrict__ b_gptq_scales,
816
- const int size_k,
817
- const int size_n,
818
- const int groups,
819
- half* __restrict__ b
820
-)
821
-{
822
- MatrixView_half_rw b_(b, size_k, size_n);
823
- MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
824
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
825
-
826
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
827
- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
828
-
829
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
830
-
831
- // Preload remapping table
832
- __shared__ int perm[BLOCK_KN_SIZE];
833
- int t = threadIdx.x;
834
-
835
- if (b_q_perm)
836
- {
837
- if (offset_k + t < size_k)
838
- perm[t] = b_q_perm[offset_k + t];
839
- }
777
+ if (b_q_perm) {
778
+ if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
779
+ }
780
841
- // Column
842
- int n = offset_n + t * 4;
843
- if (n >= size_n) return;
781
+ // Column
782
+ int n = offset_n + t * 4;
783
+ if (n >= size_n) return;
784
845
- // Find initial group
846
- int groupsize = size_k / groups;
847
- int group = offset_k / groupsize;
848
- int nextgroup = offset_k + groupsize;
785
+ // Find initial group
786
+ int groupsize = size_k / groups;
787
+ int group = offset_k / groupsize;
788
+ int nextgroup = offset_k + groupsize;
789
850
- // b offset
851
- int qk = offset_k / (32 / 8);
790
+ // b offset
791
+ int qk = offset_k / (32 / 8);
792
853
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
793
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
794
855
- // Initial zeros/scale
856
- int zeros[4];
857
- half2 scales[4];
858
- b_gptq_qzeros_.item4(zeros, group, n);
859
- b_gptq_scales_.item4_h2(scales, group, n);
795
+ // Initial zeros/scale
796
+ int zeros[4];
797
+ half2 scales[4];
798
+ b_gptq_qzeros_.item4(zeros, group, n);
799
+ b_gptq_scales_.item4_h2(scales, group, n);
800
861
- __syncthreads();
801
+ __syncthreads();
802
863
- int k = offset_k;
864
- int lk = 0;
803
+ int k = offset_k;
804
+ int lk = 0;
805
866
- while (k < end_k)
867
- {
868
- if (k == nextgroup)
869
- {
870
- group++;
871
- nextgroup += groupsize;
872
- b_gptq_qzeros_.item4(zeros, group, n);
873
- b_gptq_scales_.item4_h2(scales, group, n);
874
- }
806
+ while (k < end_k) {
807
+ if (k == nextgroup) {
808
+ group++;
809
+ nextgroup += groupsize;
810
+ b_gptq_qzeros_.item4(zeros, group, n);
811
+ b_gptq_scales_.item4_h2(scales, group, n);
812
+ }
813
876
- for (int p = 0; p < 4; p++)
877
- {
878
- int4 load_int4[2];
879
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
880
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
881
-
882
- half2 dq[4][4];
883
- dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1);
884
- dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1);
885
- dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1);
886
- dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1);
887
-
888
- //half* dqh = (half*)dq;
889
- if (b_q_perm)
890
- {
891
- for (int j = 0; j < 4; j++)
892
- {
893
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
894
- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
895
- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
896
- }
897
- }
898
- else
899
- {
900
- for (int j = 0; j < 4; j++)
901
- {
902
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
903
- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
904
- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
905
- }
906
- }
814
+ for (int p = 0; p < 4; p++) {
815
+ int4 load_int4[2];
816
+ load_int4[0] = *((int4*)b_ptr);
817
+ b_ptr += size_n;
818
+ load_int4[1] = *((int4*)b_ptr);
819
+ b_ptr += size_n;
820
+
821
+ half2 dq[4][4];
822
+ dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,
823
+ zeros[0] + 1);
824
+ dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,
825
+ zeros[1] + 1);
826
+ dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,
827
+ zeros[2] + 1);
828
+ dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
829
+ zeros[3] + 1);
830
+
831
+ // half* dqh = (half*)dq;
832
+ if (b_q_perm) {
833
+ for (int j = 0; j < 4; j++) {
834
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
835
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
836
+ __low2half(dq[2][j]), __low2half(dq[3][j]));
837
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
838
+ __high2half(dq[2][j]), __high2half(dq[3][j]));
839
}
908
- k += 32;
840
+ } else {
841
+ for (int j = 0; j < 4; j++) {
842
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
843
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
844
+ __low2half(dq[1][j]), __low2half(dq[2][j]),
845
+ __low2half(dq[3][j]));
846
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
847
+ __high2half(dq[1][j]), __high2half(dq[2][j]),
848
+ __high2half(dq[3][j]));
849
+ }
850
+ }
851
}
852
+ k += 32;
853
+ }
854
}
855
912
-__global__ void reconstruct_exllama_4bit_kernel
913
-(
914
- const uint32_t* __restrict__ b_q_weight,
915
- const int* __restrict__ b_q_perm,
856
+__global__ void reconstruct_exllama_4bit_kernel(
857
+ const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
858
const uint32_t* __restrict__ b_gptq_qzeros,
917
- const half* __restrict__ b_gptq_scales,
918
- const int size_k,
919
- const int size_n,
920
- const int groups,
921
- half* __restrict__ b
922
-)
923
-{
924
- MatrixView_half_rw b_(b, size_k, size_n);
925
- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
926
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
927
-
928
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
929
- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
930
-
931
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
932
-
933
- // Preload remapping table
934
- __shared__ int perm[BLOCK_KN_SIZE];
935
- int t = threadIdx.x;
936
-
937
- if (b_q_perm)
938
- {
939
- if (offset_k + t < size_k)
940
- perm[t] = b_q_perm[offset_k + t];
859
+ const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
860
+ const int groups, half* __restrict__ b) {
861
+ MatrixView_half_rw b_(b, size_k, size_n);
862
+ MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
863
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
864
+
865
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
866
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
867
+
868
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
869
+
870
+ // Preload remapping table
871
+ __shared__ int perm[BLOCK_KN_SIZE];
872
+ int t = threadIdx.x;
873
+
874
+ if (b_q_perm) {
875
+ if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
876
+ }
877
+
878
+ // Column
879
+ int n = offset_n + t * 4;
880
+ if (n >= size_n) return;
881
+
882
+ // Find initial group
883
+ int groupsize = size_k / groups;
884
+ int group = offset_k / groupsize;
885
+ int nextgroup = offset_k + groupsize;
886
+
887
+ // b offset
888
+ int qk = offset_k / (32 / 4);
889
+
890
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
891
+
892
+ // Initial zeros/scale
893
+ int zeros[4];
894
+ half2 scales[4];
895
+ half2 z1z16[4][2];
896
+ half2 y1y16[4][2];
897
+ b_gptq_qzeros_.item4(zeros, group, n);
898
+ b_gptq_scales_.item4_h2(scales, group, n);
899
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
900
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
901
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
902
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
903
+
904
+ __syncthreads();
905
+
906
+ int k = offset_k;
907
+ int lk = 0;
908
+
909
+ while (k < end_k) {
910
+ if (k == nextgroup) {
911
+ group++;
912
+ nextgroup += groupsize;
913
+ b_gptq_qzeros_.item4(zeros, group, n);
914
+ b_gptq_scales_.item4_h2(scales, group, n);
915
+ dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
916
+ dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
917
+ dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
918
+ dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
919
}
920
943
- // Column
944
- int n = offset_n + t * 4;
945
- if (n >= size_n) return;
946
-
947
- // Find initial group
948
- int groupsize = size_k / groups;
949
- int group = offset_k / groupsize;
950
- int nextgroup = offset_k + groupsize;
951
-
952
- // b offset
953
- int qk = offset_k / (32 / 4);
954
-
955
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
956
-
957
- // Initial zeros/scale
958
- int zeros[4];
959
- half2 scales[4];
960
- half2 z1z16[4][2];
961
- half2 y1y16[4][2];
962
- b_gptq_qzeros_.item4(zeros, group, n);
963
- b_gptq_scales_.item4_h2(scales, group, n);
964
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
965
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
966
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
967
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
968
-
969
- __syncthreads();
970
-
971
- int k = offset_k;
972
- int lk = 0;
973
-
974
- while (k < end_k)
975
- {
976
- if (k == nextgroup)
977
- {
978
- group++;
979
- nextgroup += groupsize;
980
- b_gptq_qzeros_.item4(zeros, group, n);
981
- b_gptq_scales_.item4_h2(scales, group, n);
982
- dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);
983
- dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);
984
- dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);
985
- dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);
921
+ for (int p = 0; p < 4; p++) {
922
+ half2 dq[4][4];
923
+ const int4* b_ptr4 = (int4*)b_ptr;
924
+ int4 load_int4 = *b_ptr4;
925
+
926
+ dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n,
927
+ false);
928
+ dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n,
929
+ false);
930
+ dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n,
931
+ false);
932
+ dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n,
933
+ false);
934
+
935
+ b_ptr += size_n;
936
+ // half* dqh = (half*)dq;
937
+ if (b_q_perm) {
938
+ for (int j = 0; j < 4; j++) {
939
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
940
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
941
+ __low2half(dq[2][j]), __low2half(dq[3][j]));
942
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
943
+ __high2half(dq[2][j]), __high2half(dq[3][j]));
944
}
987
-
988
- for (int p = 0; p < 4; p++)
989
- {
990
- half2 dq[4][4];
991
- const int4* b_ptr4 = (int4*) b_ptr;
992
- int4 load_int4 = *b_ptr4;
993
-
994
- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
995
- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
996
- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
997
- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
998
-
999
- b_ptr += size_n;
1000
- //half* dqh = (half*)dq;
1001
- if (b_q_perm)
1002
- {
1003
- for (int j = 0; j < 4; j++)
1004
- {
1005
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1006
- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1007
- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1008
- }
1009
- }
1010
- else
1011
- {
1012
- for (int j = 0; j < 4; j++)
1013
- {
1014
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1015
- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1016
- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1017
- }
1018
- }
945
+ } else {
946
+ for (int j = 0; j < 4; j++) {
947
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
948
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
949
+ __low2half(dq[1][j]), __low2half(dq[2][j]),
950
+ __low2half(dq[3][j]));
951
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
952
+ __high2half(dq[1][j]), __high2half(dq[2][j]),
953
+ __high2half(dq[3][j]));
954
}
1020
- k += 32;
955
+ }
956
}
957
+ k += 32;
958
+ }
959
}
960
1024
-__global__ void reconstruct_exllama_3bit_kernel
1025
-(
1026
- const uint32_t* __restrict__ b_q_weight,
1027
- const int* __restrict__ b_q_perm,
961
+__global__ void reconstruct_exllama_3bit_kernel(
962
+ const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
963
const uint32_t* __restrict__ b_gptq_qzeros,
1029
- const half* __restrict__ b_gptq_scales,
1030
- const int size_k,
1031
- const int size_n,
1032
- const int groups,
1033
- half* __restrict__ b
1034
-)
1035
-{
1036
- MatrixView_half_rw b_(b, size_k, size_n);
1037
- MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1038
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1039
-
1040
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1041
- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1042
-
1043
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1044
-
1045
- // Preload remapping table
1046
- __shared__ int perm[BLOCK_KN_SIZE];
1047
- int t = threadIdx.x;
1048
-
1049
- if (b_q_perm)
1050
- {
1051
- if (offset_k + t < size_k)
1052
- perm[t] = b_q_perm[offset_k + t];
1053
- }
964
+ const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
965
+ const int groups, half* __restrict__ b) {
966
+ MatrixView_half_rw b_(b, size_k, size_n);
967
+ MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
968
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
969
1055
- // Column
1056
- int n = offset_n + t * 4;
1057
- if (n >= size_n) return;
970
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
971
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
972
1059
- // Find initial group
1060
- int groupsize = size_k / groups;
1061
- int group = offset_k / groupsize;
1062
- int nextgroup = offset_k + groupsize;
973
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
974
1064
- // b offset
1065
- int qk = offset_k / 32* 3;
975
+ // Preload remapping table
976
+ __shared__ int perm[BLOCK_KN_SIZE];
977
+ int t = threadIdx.x;
978
1067
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
979
+ if (b_q_perm) {
980
+ if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
981
+ }
982
1069
- // Initial zeros/scale
1070
- int zeros[4];
1071
- half2 scales[4];
1072
- b_gptq_qzeros_.item4(zeros, group, n);
1073
- b_gptq_scales_.item4_h2(scales, group, n);
983
+ // Column
984
+ int n = offset_n + t * 4;
985
+ if (n >= size_n) return;
986
1075
- __syncthreads();
987
+ // Find initial group
988
+ int groupsize = size_k / groups;
989
+ int group = offset_k / groupsize;
990
+ int nextgroup = offset_k + groupsize;
991
1077
- int k = offset_k;
1078
- int lk = 0;
992
+ // b offset
993
+ int qk = offset_k / 32 * 3;
994
1080
- while (k < end_k)
1081
- {
1082
- if (k == nextgroup)
1083
- {
1084
- group++;
1085
- nextgroup += groupsize;
1086
- b_gptq_qzeros_.item4(zeros, group, n);
1087
- b_gptq_scales_.item4_h2(scales, group, n);
1088
- }
995
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
996
+
997
+ // Initial zeros/scale
998
+ int zeros[4];
999
+ half2 scales[4];
1000
+ b_gptq_qzeros_.item4(zeros, group, n);
1001
+ b_gptq_scales_.item4_h2(scales, group, n);
1002
+
1003
+ __syncthreads();
1004
1090
- for (int p = 0; p < 1; p++)
1091
- {
1092
- int4 load_int4[3];
1093
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
1094
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
1095
- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
1096
-
1097
- half2 dq[4][16];
1098
- dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1);
1099
- dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1);
1100
- dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1);
1101
- dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1);
1102
-
1103
- if (b_q_perm)
1104
- {
1105
- for (int j = 0; j < 16; j++)
1106
- {
1107
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1108
- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1109
- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1110
- }
1111
- }
1112
- else
1113
- {
1114
- for (int j = 0; j < 16; j++)
1115
- {
1116
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1117
- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1118
- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1119
- }
1120
- }
1005
+ int k = offset_k;
1006
+ int lk = 0;
1007
+
1008
+ while (k < end_k) {
1009
+ if (k == nextgroup) {
1010
+ group++;
1011
+ nextgroup += groupsize;
1012
+ b_gptq_qzeros_.item4(zeros, group, n);
1013
+ b_gptq_scales_.item4_h2(scales, group, n);
1014
+ }
1015
+
1016
+ for (int p = 0; p < 1; p++) {
1017
+ int4 load_int4[3];
1018
+ load_int4[0] = *((int4*)b_ptr);
1019
+ b_ptr += size_n;
1020
+ load_int4[1] = *((int4*)b_ptr);
1021
+ b_ptr += size_n;
1022
+ load_int4[2] = *((int4*)b_ptr);
1023
+ b_ptr += size_n;
1024
+
1025
+ half2 dq[4][16];
1026
+ dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],
1027
+ size_n, zeros[0] + 1);
1028
+ dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],
1029
+ size_n, zeros[1] + 1);
1030
+ dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],
1031
+ size_n, zeros[2] + 1);
1032
+ dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
1033
+ size_n, zeros[3] + 1);
1034
+
1035
+ if (b_q_perm) {
1036
+ for (int j = 0; j < 16; j++) {
1037
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1038
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
1039
+ __low2half(dq[2][j]), __low2half(dq[3][j]));
1040
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
1041
+ __high2half(dq[2][j]), __high2half(dq[3][j]));
1042
+ }
1043
+ } else {
1044
+ for (int j = 0; j < 16; j++) {
1045
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1046
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
1047
+ __low2half(dq[1][j]), __low2half(dq[2][j]),
1048
+ __low2half(dq[3][j]));
1049
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
1050
+ __high2half(dq[1][j]), __high2half(dq[2][j]),
1051
+ __high2half(dq[3][j]));
1052
}
1122
- k += 32;
1053
+ }
1054
}
1055
+ k += 32;
1056
+ }
1057
}
1058
1126
-__global__ void reconstruct_exllama_2bit_kernel
1127
-(
1128
- const uint32_t* __restrict__ b_q_weight,
1129
- const int* __restrict__ b_q_perm,
1059
+__global__ void reconstruct_exllama_2bit_kernel(
1060
+ const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
1061
const uint32_t* __restrict__ b_gptq_qzeros,
1131
- const half* __restrict__ b_gptq_scales,
1132
- const int size_k,
1133
- const int size_n,
1134
- const int groups,
1135
- half* __restrict__ b
1136
-)
1137
-{
1138
- MatrixView_half_rw b_(b, size_k, size_n);
1139
- MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1140
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1141
-
1142
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1143
- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1144
-
1145
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1146
-
1147
- // Preload remapping table
1148
- __shared__ int perm[BLOCK_KN_SIZE];
1149
- int t = threadIdx.x;
1150
-
1151
- if (b_q_perm)
1152
- {
1153
- if (offset_k + t < size_k)
1154
- perm[t] = b_q_perm[offset_k + t];
1155
- }
1062
+ const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
1063
+ const int groups, half* __restrict__ b) {
1064
+ MatrixView_half_rw b_(b, size_k, size_n);
1065
+ MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
1066
+ MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
1067
1157
- // Column
1158
- int n = offset_n + t * 4;
1159
- if (n >= size_n) return;
1068
+ int offset_k = BLOCK_KN_SIZE * blockIdx.y;
1069
+ int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
1070
1161
- // Find initial group
1162
- int groupsize = size_k / groups;
1163
- int group = offset_k / groupsize;
1164
- int nextgroup = offset_k + groupsize;
1071
+ int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
1072
1166
- // b offset
1167
- int qk = offset_k / (32 / 2);
1073
+ // Preload remapping table
1074
+ __shared__ int perm[BLOCK_KN_SIZE];
1075
+ int t = threadIdx.x;
1076
1169
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1077
+ if (b_q_perm) {
1078
+ if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
1079
+ }
1080
1171
- // Initial zeros/scale
1172
- int zeros[4];
1173
- half2 scales[4];
1174
- b_gptq_qzeros_.item4(zeros, group, n);
1175
- b_gptq_scales_.item4_h2(scales, group, n);
1081
+ // Column
1082
+ int n = offset_n + t * 4;
1083
+ if (n >= size_n) return;
1084
1177
- __syncthreads();
1085
+ // Find initial group
1086
+ int groupsize = size_k / groups;
1087
+ int group = offset_k / groupsize;
1088
+ int nextgroup = offset_k + groupsize;
1089
1179
- int k = offset_k;
1180
- int lk = 0;
1090
+ // b offset
1091
+ int qk = offset_k / (32 / 2);
1092
1182
- while (k < end_k)
1183
- {
1184
- if (k == nextgroup)
1185
- {
1186
- group++;
1187
- nextgroup += groupsize;
1188
- b_gptq_qzeros_.item4(zeros, group, n);
1189
- b_gptq_scales_.item4_h2(scales, group, n);
1190
- }
1093
+ const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
1094
1192
- for (int p = 0; p < 2; p++)
1193
- {
1194
- const int4* b_ptr4 = (int4*) b_ptr;
1195
- int4 load_int4 = *b_ptr4;
1196
-
1197
- half2 dq[4][8];
1198
- dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1199
- dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1200
- dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1201
- dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1202
-
1203
- b_ptr += size_n;
1204
- //half* dqh = (half*)dq;
1205
- if (b_q_perm)
1206
- {
1207
- for (int j = 0; j < 8; j++)
1208
- {
1209
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1210
- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1211
- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1212
- }
1213
- }
1214
- else
1215
- {
1216
- for (int j = 0; j < 8; j++)
1217
- {
1218
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1219
- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
1220
- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
1221
- }
1222
- }
1223
- }
1224
- k += 32;
1225
- }
1226
-}
1095
+ // Initial zeros/scale
1096
+ int zeros[4];
1097
+ half2 scales[4];
1098
+ b_gptq_qzeros_.item4(zeros, group, n);
1099
+ b_gptq_scales_.item4_h2(scales, group, n);
1100
1228
-void reconstruct_exllama
1229
-(
1230
- const uint32_t* b_q_weight,
1231
- const uint32_t* b_gptq_qzeros,
1232
- const half* b_gptq_scales,
1233
- const int* b_q_perm,
1234
- half* out,
1235
- int height,
1236
- int width,
1237
- int groups,
1238
- int bit
1239
-)
1240
-{
1241
- dim3 blockDim, gridDim;
1242
- blockDim.x = BLOCK_KN_SIZE;
1243
- blockDim.y = 1;
1244
- gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1245
- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1101
+ __syncthreads();
1102
1247
- auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1248
- if (bit == 2) {
1249
- reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1250
- } else if (bit == 3) {
1251
- reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1252
- } else if (bit == 8) {
1253
- reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1103
+ int k = offset_k;
1104
+ int lk = 0;
1105
+
1106
+ while (k < end_k) {
1107
+ if (k == nextgroup) {
1108
+ group++;
1109
+ nextgroup += groupsize;
1110
+ b_gptq_qzeros_.item4(zeros, group, n);
1111
+ b_gptq_scales_.item4_h2(scales, group, n);
1112
}
1113
1256
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1257
- reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
1258
- (
1259
- b_q_weight,
1260
- b_q_perm,
1261
- b_gptq_qzeros,
1262
- b_gptq_scales,
1263
- height,
1264
- width,
1265
- groups,
1266
- out
1267
- );
1114
+ for (int p = 0; p < 2; p++) {
1115
+ const int4* b_ptr4 = (int4*)b_ptr;
1116
+ int4 load_int4 = *b_ptr4;
1117
+
1118
+ half2 dq[4][8];
1119
+ dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);
1120
+ dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);
1121
+ dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);
1122
+ dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);
1123
+
1124
+ b_ptr += size_n;
1125
+ // half* dqh = (half*)dq;
1126
+ if (b_q_perm) {
1127
+ for (int j = 0; j < 8; j++) {
1128
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1129
+ b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
1130
+ __low2half(dq[2][j]), __low2half(dq[3][j]));
1131
+ b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
1132
+ __high2half(dq[2][j]), __high2half(dq[3][j]));
1133
+ }
1134
+ } else {
1135
+ for (int j = 0; j < 8; j++) {
1136
+ for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
1137
+ b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
1138
+ __low2half(dq[1][j]), __low2half(dq[2][j]),
1139
+ __low2half(dq[3][j]));
1140
+ b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
1141
+ __high2half(dq[1][j]), __high2half(dq[2][j]),
1142
+ __high2half(dq[3][j]));
1143
+ }
1144
+ }
1145
+ }
1146
+ k += 32;
1147
+ }
1148
}
1149
1150
+void reconstruct_exllama(const uint32_t* b_q_weight,
1151
+ const uint32_t* b_gptq_qzeros,
1152
+ const half* b_gptq_scales, const int* b_q_perm,
1153
+ half* out, int height, int width, int groups,
1154
+ int bit) {
1155
+ dim3 blockDim, gridDim;
1156
+ blockDim.x = BLOCK_KN_SIZE;
1157
+ blockDim.y = 1;
1158
+ gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
1159
+ gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1160
+
1161
+ auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel;
1162
+ if (bit == 2) {
1163
+ reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel;
1164
+ } else if (bit == 3) {
1165
+ reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel;
1166
+ } else if (bit == 8) {
1167
+ reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel;
1168
+ }
1169
+
1170
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1171
+ reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(
1172
+ b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,
1173
+ out);
1174
+}
1175
1176
__global__ void gemm_half_q_half_alt_4bit_kernel(
1272
- const half2* __restrict__ vec,
1273
- const uint32_t* __restrict__ mat,
1274
- half* __restrict__ mul,
1275
- const half* __restrict__ scales,
1276
- const uint32_t* __restrict__ zeros,
1277
- const int* __restrict__ g_idx,
1278
- int batch,
1279
- int height,
1280
- int width
1281
-)
1282
-{
1283
- int zero_width = width / 8;
1284
- int vec_height = height * 4;
1285
- const int blockwidth2 = BLOCK_KN_SIZE / 2;
1286
- int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1287
- int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1288
- int h = BLOCK_KN_SIZE * blockIdx.z / 8;
1289
- int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
1290
- int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1291
-
1292
- __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1293
- if (threadIdx.x < h_end) {
1294
- for (int m = 0; m < b_end; ++m) {
1295
- blockvec[m][threadIdx.x] =
1296
- vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1297
- threadIdx.x];
1298
- }
1177
+ const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
1178
+ half* __restrict__ mul, const half* __restrict__ scales,
1179
+ const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
1180
+ int batch, int height, int width) {
1181
+ int zero_width = width / 8;
1182
+ int vec_height = height * 4;
1183
+ const int blockwidth2 = BLOCK_KN_SIZE / 2;
1184
+ int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1185
+ int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1186
+ int h = BLOCK_KN_SIZE * blockIdx.z / 8;
1187
+ int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;
1188
+ int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1189
+
1190
+ __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1191
+ if (threadIdx.x < h_end) {
1192
+ for (int m = 0; m < b_end; ++m) {
1193
+ blockvec[m][threadIdx.x] =
1194
+ vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1195
+ threadIdx.x];
1196
}
1300
-
1301
- __shared__ half2 deq2[256][8];
1302
- int val = threadIdx.x / 8;
1303
- int off = threadIdx.x % 8;
1304
- for (; val < 256; val += BLOCK_KN_SIZE / 8) {
1305
- deq2[val][off] = __halves2half2(
1306
- __int2half_rn(val & 0xF), __int2half_rn(val >> 4)
1307
- );
1197
+ }
1198
+
1199
+ __shared__ half2 deq2[256][8];
1200
+ int val = threadIdx.x / 8;
1201
+ int off = threadIdx.x % 8;
1202
+ for (; val < 256; val += BLOCK_KN_SIZE / 8) {
1203
+ deq2[val][off] =
1204
+ __halves2half2(__int2half_rn(val & 0xF), __int2half_rn(val >> 4));
1205
+ }
1206
+
1207
+ if (blockIdx.z == 0) {
1208
+ for (int m = 0; m < b_end; m++) mul[(b + m) * width + w] = __int2half_rn(0);
1209
+ }
1210
+ __syncthreads();
1211
+
1212
+ int i = width * h + w;
1213
+ int g_h = h * 8;
1214
+ int k = 0;
1215
+ int z_w = w / 8;
1216
+ int z_mod = (w % 8) * 4;
1217
+ half2 res2;
1218
+ half res[BLOCK_M_SIZE_MAX] = {};
1219
+
1220
+ unsigned int tmp;
1221
+ while (k < h_end) {
1222
+ tmp = mat[i];
1223
+ half2 scales_tmp[4];
1224
+ half2 zeros_tmp[4];
1225
+ for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
1226
+ int g = g_idx[g_h + (k + tmp_k) * 2];
1227
+ int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1228
+ half scale_f = scales[g * width + w];
1229
+ half scale_f2 = scales[g2 * width + w];
1230
+ half2 scale = __halves2half2(scale_f, scale_f2);
1231
+ half2 zero = __halves2half2(
1232
+ __hmul(scale_f,
1233
+ __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) -
1234
+ 1)),
1235
+ __hmul(scale_f2,
1236
+ __int2half_rn(
1237
+ -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1)));
1238
+ scales_tmp[tmp_k] = scale;
1239
+ zeros_tmp[tmp_k] = zero;
1240
}
1309
-
1310
- if (blockIdx.z == 0)
1311
- {
1312
- for (int m = 0; m < b_end; m++)
1313
- mul[(b + m) * width + w] = __int2half_rn(0);
1314
- }
1315
- __syncthreads();
1316
-
1317
- int i = width * h + w;
1318
- int g_h = h * 8;
1319
- int k = 0;
1320
- int z_w = w / 8;
1321
- int z_mod = (w % 8) * 4;
1322
- half2 res2;
1323
- half res[BLOCK_M_SIZE_MAX] = {};
1324
-
1325
- unsigned int tmp;
1326
- while (k < h_end) {
1327
- tmp = mat[i];
1328
- half2 scales_tmp[4];
1329
- half2 zeros_tmp[4];
1330
- for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
1331
- int g = g_idx[g_h + (k + tmp_k) * 2];
1332
- int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1333
- half scale_f = scales[g * width + w];
1334
- half scale_f2 = scales[g2 * width + w];
1335
- half2 scale = __halves2half2(scale_f, scale_f2);
1336
- half2 zero = __halves2half2(
1337
- __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)),
1338
- __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))
1339
- );
1340
- scales_tmp[tmp_k] = scale;
1341
- zeros_tmp[tmp_k] = zero;
1342
- }
1343
- for (int m = 0; m < b_end; m++) {
1241
+ for (int m = 0; m < b_end; m++) {
1242
#ifndef USE_ROCM
1345
- res2 = {};
1243
+ res2 = {};
1244
#else
1347
- res2.x = __half_as_ushort(__float2half(0));
1348
- res2.y = __half_as_ushort(__float2half(0));
1245
+ res2.x = __half_as_ushort(__float2half(0));
1246
+ res2.y = __half_as_ushort(__float2half(0));
1247
#endif
1350
- res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1351
- res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1352
- res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2);
1353
- res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2);
1248
+ res2 = __hfma2(
1249
+ __hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]),
1250
+ blockvec[m][k + 0], res2);
1251
+ res2 = __hfma2(
1252
+ __hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]),
1253
+ blockvec[m][k + 1], res2);
1254
+ res2 = __hfma2(
1255
+ __hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]),
1256
+ blockvec[m][k + 2], res2);
1257
+ res2 = __hfma2(
1258
+ __hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]),
1259
+ blockvec[m][k + 3], res2);
1260
#ifndef USE_ROCM
1355
- res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1261
+ res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1262
#else
1357
- res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1263
+ res[m] = __hadd(
1264
+ res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1265
#endif
1359
- }
1360
- i += width;
1361
- k += 4;
1362
- }
1363
- for (int m = 0; m < b_end; m++) {
1364
- atomicAdd(&mul[(b + m) * width + w], res[m]);
1266
}
1267
+ i += width;
1268
+ k += 4;
1269
+ }
1270
+ for (int m = 0; m < b_end; m++) {
1271
+ atomicAdd(&mul[(b + m) * width + w], res[m]);
1272
+ }
1273
}
1274
1368
-
1275
__global__ void gemm_half_q_half_alt_8bit_kernel(
1370
- const half2* __restrict__ vec,
1371
- const uint32_t* __restrict__ mat,
1372
- half* __restrict__ mul,
1373
- const half* __restrict__ scales,
1374
- const uint32_t* __restrict__ zeros,
1375
- const int* __restrict__ g_idx,
1376
- int batch,
1377
- int height,
1378
- int width
1379
-)
1380
-{
1381
- int zero_width = width / 4;
1382
- int vec_height = height * 2;
1383
- const int blockwidth2 = BLOCK_KN_SIZE / 2;
1384
- int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1385
- int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1386
- int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1387
- int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1388
- int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1389
-
1390
- __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1391
- if (threadIdx.x < h_end) {
1392
- for (int m = 0; m < b_end; ++m) {
1393
- blockvec[m][threadIdx.x] =
1394
- vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1395
- threadIdx.x];
1396
- }
1276
+ const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
1277
+ half* __restrict__ mul, const half* __restrict__ scales,
1278
+ const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
1279
+ int batch, int height, int width) {
1280
+ int zero_width = width / 4;
1281
+ int vec_height = height * 2;
1282
+ const int blockwidth2 = BLOCK_KN_SIZE / 2;
1283
+ int b = blockIdx.y * BLOCK_M_SIZE_MAX;
1284
+ int b_end = min(BLOCK_M_SIZE_MAX, batch - b);
1285
+ int h = BLOCK_KN_SIZE * blockIdx.z / 4;
1286
+ int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;
1287
+ int w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1288
+
1289
+ __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];
1290
+ if (threadIdx.x < h_end) {
1291
+ for (int m = 0; m < b_end; ++m) {
1292
+ blockvec[m][threadIdx.x] =
1293
+ vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 +
1294
+ threadIdx.x];
1295
}
1398
-
1399
-
1400
- if (blockIdx.z == 0)
1401
- {
1402
- for (int m = 0; m < b_end; m++)
1403
- mul[(b + m) * width + w] = __int2half_rn(0);
1296
+ }
1297
+
1298
+ if (blockIdx.z == 0) {
1299
+ for (int m = 0; m < b_end; m++) mul[(b + m) * width + w] = __int2half_rn(0);
1300
+ }
1301
+ __syncthreads();
1302
+
1303
+ int i = width * h + w;
1304
+ int g_h = h * 4;
1305
+ int k = 0;
1306
+ int z_w = w / 4;
1307
+ int z_mod = (w % 4) * 8;
1308
+ half2 res2;
1309
+ half res[BLOCK_M_SIZE_MAX] = {};
1310
+
1311
+ unsigned int tmp;
1312
+ while (k < h_end) {
1313
+ tmp = mat[i];
1314
+ half2 scales_tmp[2];
1315
+ half2 zeros_tmp[2];
1316
+ for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1317
+ int g = g_idx[g_h + (k + tmp_k) * 2];
1318
+ int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1319
+ half scale_f = scales[g * width + w];
1320
+ half scale_f2 = scales[g2 * width + w];
1321
+ half2 scale = __halves2half2(scale_f, scale_f2);
1322
+ half2 zero = __halves2half2(
1323
+ __hmul(scale_f,
1324
+ __int2half_rn(
1325
+ -((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1326
+ __hmul(scale_f2,
1327
+ __int2half_rn(
1328
+ -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1)));
1329
+ scales_tmp[tmp_k] = scale;
1330
+ zeros_tmp[tmp_k] = zero;
1331
}
1405
- __syncthreads();
1406
-
1407
- int i = width * h + w;
1408
- int g_h = h * 4;
1409
- int k = 0;
1410
- int z_w = w / 4;
1411
- int z_mod = (w % 4) * 8;
1412
- half2 res2;
1413
- half res[BLOCK_M_SIZE_MAX] = {};
1414
-
1415
- unsigned int tmp;
1416
- while (k < h_end) {
1417
- tmp = mat[i];
1418
- half2 scales_tmp[2];
1419
- half2 zeros_tmp[2];
1420
- for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
1421
- int g = g_idx[g_h + (k + tmp_k) * 2];
1422
- int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
1423
- half scale_f = scales[g * width + w];
1424
- half scale_f2 = scales[g2 * width + w];
1425
- half2 scale = __halves2half2(scale_f, scale_f2);
1426
- half2 zero = __halves2half2(
1427
- __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),
1428
- __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))
1429
- );
1430
- scales_tmp[tmp_k] = scale;
1431
- zeros_tmp[tmp_k] = zero;
1432
- }
1433
- for (int m = 0; m < b_end; m++) {
1332
+ for (int m = 0; m < b_end; m++) {
1333
#ifndef USE_ROCM
1435
- res2 = {};
1334
+ res2 = {};
1335
#else
1437
- res2.x = __half_as_ushort(__float2half(0));
1438
- res2.y = __half_as_ushort(__float2half(0));
1336
+ res2.x = __half_as_ushort(__float2half(0));
1337
+ res2.y = __half_as_ushort(__float2half(0));
1338
#endif
1440
- half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF));
1441
- res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2);
1442
- half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF));
1443
- res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2);
1339
+ half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF),
1340
+ __int2half_rn((tmp >> 8) & 0xFF));
1341
+ res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]),
1342
+ blockvec[m][k + 0], res2);
1343
+ half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF),
1344
+ __int2half_rn((tmp >> 24) & 0xFF));
1345
+ res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]),
1346
+ blockvec[m][k + 1], res2);
1347
#ifndef USE_ROCM
1445
- res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1348
+ res[m] = __hadd(res[m], __hadd(res2.x, res2.y));
1349
#else
1447
- res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1350
+ res[m] = __hadd(
1351
+ res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)));
1352
#endif
1449
- }
1450
- i += width;
1451
- k += 2;
1452
- }
1453
- for (int m = 0; m < b_end; m++) {
1454
- atomicAdd(&mul[(b + m) * width + w], res[m]);
1353
}
1354
+ i += width;
1355
+ k += 2;
1356
+ }
1357
+ for (int m = 0; m < b_end; m++) {
1358
+ atomicAdd(&mul[(b + m) * width + w], res[m]);
1359
+ }
1360
}
1361
1458
-void gemm_half_q_half_alt
1459
-(
1460
- const half* a,
1461
- const uint32_t* b_q_weight,
1462
- const uint32_t* b_gptq_qzeros,
1463
- const half* b_gptq_scales,
1464
- const int* b_g_idx,
1465
- half* c,
1466
- int size_m,
1467
- int size_n,
1468
- int size_k,
1469
- int bit
1470
-)
1471
-{
1472
- dim3 blockDim, gridDim;
1473
- blockDim.x = BLOCK_KN_SIZE;
1474
- blockDim.y = 1;
1475
- blockDim.z = 1;
1476
- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
1477
- gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1478
- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1479
-
1480
- auto kernel = gemm_half_q_half_alt_4bit_kernel;
1481
- if (bit == 8) {
1482
- kernel = gemm_half_q_half_alt_8bit_kernel;
1483
- }
1484
-
1485
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1486
- kernel<<<gridDim, blockDim, 0, stream>>>
1487
- (
1488
- (const half2*) a,
1489
- b_q_weight,
1490
- c,
1491
- b_gptq_scales,
1492
- b_gptq_qzeros,
1493
- b_g_idx,
1494
- size_m,
1495
- size_k / 32 * bit,
1496
- size_n
1497
- );
1362
+void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
1363
+ const uint32_t* b_gptq_qzeros,
1364
+ const half* b_gptq_scales, const int* b_g_idx,
1365
+ half* c, int size_m, int size_n, int size_k,
1366
+ int bit) {
1367
+ dim3 blockDim, gridDim;
1368
+ blockDim.x = BLOCK_KN_SIZE;
1369
+ blockDim.y = 1;
1370
+ blockDim.z = 1;
1371
+ gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
1372
+ gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
1373
+ gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
1374
+
1375
+ auto kernel = gemm_half_q_half_alt_4bit_kernel;
1376
+ if (bit == 8) {
1377
+ kernel = gemm_half_q_half_alt_8bit_kernel;
1378
+ }
1379
+
1380
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1381
+ kernel<<<gridDim, blockDim, 0, stream>>>(
1382
+ (const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,
1383
+ size_m, size_k / 32 * bit, size_n);
1384
}
1385
1500
-template<class T, int bit>
1501
-__global__ void reconstruct_gptq_kernel
1502
-(
1503
- const uint32_t* __restrict__ w,
1504
- const half* __restrict__ w_scales,
1505
- const uint32_t* __restrict__ w_zeros,
1506
- const int* __restrict__ g_idx,
1507
- const int height,
1508
- const int width,
1509
- const int group,
1510
- half* __restrict__ out
1511
-)
1512
-{
1513
- // Start of block
1514
-
1515
- int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1516
- int row = blockIdx.y * 32 / bit;
1517
- if (column >= width) return;
1518
-
1519
- // Views
1520
-
1521
- MatrixView_half_rw out_(out, height, width);
1522
- MatrixView_half w_scales_(w_scales, group, width);
1523
- T w_zeros_(w_zeros, group, width);
1524
-
1525
- uint32_t w_read = w[blockIdx.y * width + column];
1526
- half* out_ptr = out_.item_ptr(row, column);
1527
-
1528
- #pragma unroll
1529
- for (int s = 0; s < 32; s += bit)
1530
- {
1531
- int group = g_idx[row + s / bit];
1532
- half w_scale = w_scales_.item(group, column);
1533
- uint32_t w_zero = w_zeros_.item(group, column) + 1;
1534
- half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale);
1535
- *out_ptr = w_item; out_ptr += out_.width;
1536
- }
1386
+template <class T, int bit>
1387
+__global__ void reconstruct_gptq_kernel(const uint32_t* __restrict__ w,
1388
+ const half* __restrict__ w_scales,
1389
+ const uint32_t* __restrict__ w_zeros,
1390
+ const int* __restrict__ g_idx,
1391
+ const int height, const int width,
1392
+ const int group,
1393
+ half* __restrict__ out) {
1394
+ // Start of block
1395
+
1396
+ int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1397
+ int row = blockIdx.y * 32 / bit;
1398
+ if (column >= width) return;
1399
+
1400
+ // Views
1401
+
1402
+ MatrixView_half_rw out_(out, height, width);
1403
+ MatrixView_half w_scales_(w_scales, group, width);
1404
+ T w_zeros_(w_zeros, group, width);
1405
+
1406
+ uint32_t w_read = w[blockIdx.y * width + column];
1407
+ half* out_ptr = out_.item_ptr(row, column);
1408
+
1409
+#pragma unroll
1410
+ for (int s = 0; s < 32; s += bit) {
1411
+ int group = g_idx[row + s / bit];
1412
+ half w_scale = w_scales_.item(group, column);
1413
+ uint32_t w_zero = w_zeros_.item(group, column) + 1;
1414
+ half w_item =
1415
+ __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero),
1416
+ w_scale);
1417
+ *out_ptr = w_item;
1418
+ out_ptr += out_.width;
1419
+ }
1420
}
1421
1539
-__global__ void reconstruct_gptq_3bit_kernel
1540
-(
1541
- const uint32_t* __restrict__ w,
1542
- const half* __restrict__ w_scales,
1543
- const uint32_t* __restrict__ w_zeros,
1544
- const int* __restrict__ g_idx,
1545
- const int height,
1546
- const int width,
1547
- const int group,
1548
- half* __restrict__ out
1549
-)
1550
-{
1551
- // Start of block
1552
- int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1553
- int row = blockIdx.y * 32;
1554
- if (column >= width) return;
1555
-
1556
- // Views
1557
-
1558
- MatrixView_half_rw out_(out, height, width);
1559
- MatrixView_half w_scales_(w_scales, group, width);
1560
- MatrixView_q3_row w_zeros_(w_zeros, group, width);
1561
-
1562
- uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1563
- uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1564
- uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1565
- half* out_ptr = out_.item_ptr(row, column);
1566
-
1567
- #pragma unroll
1568
- for (int i = 0; i < 32; i += 1)
1569
- {
1570
- int group = g_idx[row + i];
1571
- half w_scale = w_scales_.item(group, column);
1572
- uint32_t w_zero = w_zeros_.item(group, column) + 1;
1573
- int w_item;
1574
- if (i == 10) {
1575
- w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1576
- } else if (i == 21) {
1577
- w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1578
- } else if (i < 10) {
1579
- w_item = ((w1 >> (i * 3)) & 0x7);
1580
- } else if (i < 21) {
1581
- w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1582
- } else {
1583
- w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1584
- }
1585
- *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1586
- out_ptr += out_.width;
1422
+__global__ void reconstruct_gptq_3bit_kernel(
1423
+ const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
1424
+ const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
1425
+ const int height, const int width, const int group,
1426
+ half* __restrict__ out) {
1427
+ // Start of block
1428
+ int column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
1429
+ int row = blockIdx.y * 32;
1430
+ if (column >= width) return;
1431
+
1432
+ // Views
1433
+
1434
+ MatrixView_half_rw out_(out, height, width);
1435
+ MatrixView_half w_scales_(w_scales, group, width);
1436
+ MatrixView_q3_row w_zeros_(w_zeros, group, width);
1437
+
1438
+ uint32_t w1 = w[(blockIdx.y * 3) * width + column];
1439
+ uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];
1440
+ uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];
1441
+ half* out_ptr = out_.item_ptr(row, column);
1442
+
1443
+#pragma unroll
1444
+ for (int i = 0; i < 32; i += 1) {
1445
+ int group = g_idx[row + i];
1446
+ half w_scale = w_scales_.item(group, column);
1447
+ uint32_t w_zero = w_zeros_.item(group, column) + 1;
1448
+ int w_item;
1449
+ if (i == 10) {
1450
+ w_item = (w1 >> 30) | ((w2 << 2) & 0x4);
1451
+ } else if (i == 21) {
1452
+ w_item = (w2 >> 31) | ((w3 << 1) & 0x6);
1453
+ } else if (i < 10) {
1454
+ w_item = ((w1 >> (i * 3)) & 0x7);
1455
+ } else if (i < 21) {
1456
+ w_item = ((w2 >> (i * 3 - 32)) & 0x7);
1457
+ } else {
1458
+ w_item = ((w3 >> (i * 3 - 64)) & 0x7);
1459
}
1460
+ *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale);
1461
+ out_ptr += out_.width;
1462
+ }
1463
}
1464
1590
-void reconstruct_gptq
1591
-(
1592
- const uint32_t* b_q_weight,
1593
- const uint32_t* b_gptq_qzeros,
1594
- const half* b_gptq_scales,
1595
- const int* b_g_idx,
1596
- half* out,
1597
- int height,
1598
- int width,
1599
- int groups,
1600
- int bit
1601
-)
1602
-{
1603
- dim3 blockDim, gridDim;
1604
- blockDim.x = BLOCK_KN_SIZE;
1605
- blockDim.y = 1;
1606
- gridDim.y = DIVIDE(height, 32 / bit);
1607
- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1608
-
1609
- auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1610
- if (bit == 2) {
1611
- kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1612
- } else if (bit == 8) {
1613
- kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1614
- } else if (bit == 3) {
1615
- kernel = reconstruct_gptq_3bit_kernel;
1616
- gridDim.y = DIVIDE(height, 32);
1617
- }
1618
-
1619
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1620
- kernel<<<gridDim, blockDim, 0, stream>>>
1621
- (
1622
- b_q_weight,
1623
- b_gptq_scales,
1624
- b_gptq_qzeros,
1625
- b_g_idx,
1626
- height,
1627
- width,
1628
- groups,
1629
- out
1630
- );
1465
+void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
1466
+ const half* b_gptq_scales, const int* b_g_idx, half* out,
1467
+ int height, int width, int groups, int bit) {
1468
+ dim3 blockDim, gridDim;
1469
+ blockDim.x = BLOCK_KN_SIZE;
1470
+ blockDim.y = 1;
1471
+ gridDim.y = DIVIDE(height, 32 / bit);
1472
+ gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
1473
+
1474
+ auto kernel = reconstruct_gptq_kernel<MatrixView_q4_row, 4>;
1475
+ if (bit == 2) {
1476
+ kernel = reconstruct_gptq_kernel<MatrixView_q2_row, 2>;
1477
+ } else if (bit == 8) {
1478
+ kernel = reconstruct_gptq_kernel<MatrixView_q8_row, 8>;
1479
+ } else if (bit == 3) {
1480
+ kernel = reconstruct_gptq_3bit_kernel;
1481
+ gridDim.y = DIVIDE(height, 32);
1482
+ }
1483
+
1484
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1485
+ kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,
1486
+ b_gptq_qzeros, b_g_idx, height,
1487
+ width, groups, out);
1488
}
1489
1633
-
1634
-void gemm_half_q_half_cuda
1635
-(
1636
- cublasHandle_t cublas_handle,
1637
- const half* a,
1638
- const uint32_t* b_q_weight,
1639
- const uint32_t* b_gptq_qzeros,
1640
- const half* b_gptq_scales,
1641
- const int* b_g_idx,
1642
- half* c,
1643
- half* temp_dq,
1644
- int size_m,
1645
- int size_n,
1646
- int size_k,
1647
- int groups,
1648
- bool use_exllama,
1649
- int bit
1650
-)
1651
-{
1652
- bool use_reconstruct;
1490
+void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
1491
+ const uint32_t* b_q_weight,
1492
+ const uint32_t* b_gptq_qzeros,
1493
+ const half* b_gptq_scales, const int* b_g_idx,
1494
+ half* c, half* temp_dq, int size_m, int size_n,
1495
+ int size_k, int groups, bool use_exllama, int bit) {
1496
+ bool use_reconstruct;
1497
+ if (use_exllama) {
1498
+ use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||
1499
+ (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1500
+ } else {
1501
+ // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so
1502
+ // we disabled them for now.
1503
+ use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1504
+ }
1505
+ if (use_reconstruct) {
1506
+ // Reconstruct FP16 matrix, then cuBLAS
1507
if (use_exllama) {
1654
- use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS));
1508
+ reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1509
+ temp_dq, size_k, size_n, groups, bit);
1510
} else {
1656
- // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so we disabled them for now.
1657
- use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS);
1511
+ reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1512
+ temp_dq, size_k, size_n, groups, bit);
1513
}
1659
- if (use_reconstruct) {
1660
- // Reconstruct FP16 matrix, then cuBLAS
1661
- if (use_exllama) {
1662
- reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq,
1663
- size_k, size_n, groups, bit);
1664
- }
1665
- else
1666
- {
1667
- reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1668
- temp_dq, size_k, size_n, groups, bit);
1669
- }
1514
1671
- const half alpha = __float2half(1.0f);
1672
- const half beta = __float2half(0.0f);
1673
- cublasHgemm(cublas_handle,
1674
- CUBLAS_OP_N,
1675
- CUBLAS_OP_N,
1676
- size_n, size_m, size_k,
1677
- &alpha, temp_dq, size_n,
1678
- a, size_k,
1679
- &beta, c, size_n);
1515
+ const half alpha = __float2half(1.0f);
1516
+ const half beta = __float2half(0.0f);
1517
+ cublasHgemm(cublas_handle, CUBLAS_OP_N, CUBLAS_OP_N, size_n, size_m, size_k,
1518
+ &alpha, temp_dq, size_n, a, size_k, &beta, c, size_n);
1519
+ } else if (use_exllama) {
1520
+ // Quantized matmul
1521
+ int max_chunks = size_m / BLOCK_M_SIZE_MAX;
1522
+ int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
1523
+ int last_chunk_size = size_m - last_chunk;
1524
+
1525
+ if (max_chunks) {
1526
+ gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
1527
+ b_g_idx, c, last_chunk, size_n, size_k,
1528
+ BLOCK_M_SIZE_MAX, groups, bit);
1529
}
1681
- else if (use_exllama)
1682
- {
1683
- // Quantized matmul
1684
- int max_chunks = size_m / BLOCK_M_SIZE_MAX;
1685
- int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
1686
- int last_chunk_size = size_m - last_chunk;
1687
-
1688
- if (max_chunks)
1689
- {
1690
- gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1691
- c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
1692
- groups, bit);
1693
- }
1530
1695
- if (last_chunk_size)
1696
- {
1697
- gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight, b_gptq_qzeros,
1698
- b_gptq_scales, b_g_idx, c + last_chunk * size_n,
1699
- last_chunk_size, size_n, size_k, last_chunk_size,
1700
- groups, bit);
1701
- }
1702
- }
1703
- else
1704
- {
1705
- gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1706
- c, size_m, size_n, size_k, bit);
1531
+ if (last_chunk_size) {
1532
+ gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight,
1533
+ b_gptq_qzeros, b_gptq_scales, b_g_idx,
1534
+ c + last_chunk * size_n, last_chunk_size,
1535
+ size_n, size_k, last_chunk_size, groups, bit);
1536
}
1537
+ } else {
1538
+ gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
1539
+ c, size_m, size_n, size_k, bit);
1540
+ }
1541
}
1542
1710
-__global__ void shuffle_4bit_kernel
1711
-(
1712
- uint32_t* __restrict__ b_q_weight,
1713
- const int size_k,
1714
- const int size_n
1715
-)
1716
-{
1717
- int n = blockIdx.x * THREADS_X + threadIdx.x;
1718
- if (n >= size_n) return;
1719
- int k = 0;
1720
- uint32_t* b_ptr = b_q_weight + n;
1721
- while (k < size_k) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
1543
+__global__ void shuffle_4bit_kernel(uint32_t* __restrict__ b_q_weight,
1544
+ const int size_k, const int size_n) {
1545
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1546
+ if (n >= size_n) return;
1547
+ int k = 0;
1548
+ uint32_t* b_ptr = b_q_weight + n;
1549
+ while (k < size_k) {
1550
+ shuffle_4bit_8(b_ptr, size_n);
1551
+ b_ptr += 1 * size_n;
1552
+ k += 8;
1553
+ }
1554
}
1555
1724
-__global__ void shuffle_8bit_kernel
1725
-(
1726
- uint32_t* __restrict__ b_q_weight,
1727
- const int size_k,
1728
- const int size_n
1729
-)
1730
-{
1731
- int n = blockIdx.x * THREADS_X + threadIdx.x;
1732
- if (n >= size_n) return;
1733
- int k = 0;
1734
- uint32_t* b_ptr = b_q_weight + n;
1735
- while (k < size_k) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }
1556
+__global__ void shuffle_8bit_kernel(uint32_t* __restrict__ b_q_weight,
1557
+ const int size_k, const int size_n) {
1558
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1559
+ if (n >= size_n) return;
1560
+ int k = 0;
1561
+ uint32_t* b_ptr = b_q_weight + n;
1562
+ while (k < size_k) {
1563
+ shuffle_8bit_4(b_ptr, size_n);
1564
+ b_ptr += 1 * size_n;
1565
+ k += 4;
1566
+ }
1567
}
1568
1738
-__global__ void shuffle_2bit_kernel
1739
-(
1740
- uint32_t* __restrict__ b_q_weight,
1741
- const int size_k,
1742
- const int size_n
1743
-)
1744
-{
1745
- int n = blockIdx.x * THREADS_X + threadIdx.x;
1746
- if (n >= size_n) return;
1747
- int k = 0;
1748
- uint32_t* b_ptr = b_q_weight + n;
1749
- while (k < size_k) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }
1569
+__global__ void shuffle_2bit_kernel(uint32_t* __restrict__ b_q_weight,
1570
+ const int size_k, const int size_n) {
1571
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1572
+ if (n >= size_n) return;
1573
+ int k = 0;
1574
+ uint32_t* b_ptr = b_q_weight + n;
1575
+ while (k < size_k) {
1576
+ shuffle_2bit_16(b_ptr, size_n);
1577
+ b_ptr += 1 * size_n;
1578
+ k += 16;
1579
+ }
1580
}
1581
1752
-__global__ void shuffle_3bit_kernel
1753
-(
1754
- uint32_t* __restrict__ b_q_weight,
1755
- const int size_k,
1756
- const int size_n
1757
-)
1758
-{
1759
- int n = blockIdx.x * THREADS_X + threadIdx.x;
1760
- if (n >= size_n) return;
1761
- int k = 0;
1762
- uint32_t* b_ptr = b_q_weight + n;
1763
- while (k < size_k) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }
1582
+__global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight,
1583
+ const int size_k, const int size_n) {
1584
+ int n = blockIdx.x * THREADS_X + threadIdx.x;
1585
+ if (n >= size_n) return;
1586
+ int k = 0;
1587
+ uint32_t* b_ptr = b_q_weight + n;
1588
+ while (k < size_k) {
1589
+ shuffle_3bit_32(b_ptr, size_n);
1590
+ b_ptr += 3 * size_n;
1591
+ k += 32;
1592
+ }
1593
}
1594
1766
-__global__ void make_sequential_4bit_kernel
1767
-(
1768
- const uint32_t* __restrict__ w,
1769
- uint32_t* __restrict__ w_new,
1770
- const int* __restrict__ q_perm,
1771
- const int w_width
1772
-)
1773
-{
1774
- const uint64_t* w2 = (uint64_t*) w;
1775
- uint64_t* w_new2 = (uint64_t*) w_new;
1776
- int w2_stride = w_width >> 1;
1777
- int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1778
- if (w2_column >= w2_stride) return;
1779
- int w_new2_row = blockIdx.y;
1780
- int q_perm_idx = w_new2_row << 3;
1781
- uint64_t dst = 0;
1782
-
1783
- #pragma unroll
1784
- for (int i = 0; i < 8; i++)
1785
- {
1786
- int source_row = q_perm[q_perm_idx++];
1787
-
1788
- int w2_row = source_row >> 3;
1789
- int w2_subrow = source_row & 0x07;
1790
- int w2_row_shift = w2_subrow << 2;
1791
- int wnew2_row_shift = i << 2;
1792
-
1793
- uint64_t src = w2[w2_row * w2_stride + w2_column];
1794
- src >>= w2_row_shift;
1795
- src &= 0x0000000f0000000f;
1796
- src <<= wnew2_row_shift;
1797
- dst |= src;
1798
- }
1799
- w_new2[w_new2_row * w2_stride + w2_column] = dst;
1595
+__global__ void make_sequential_4bit_kernel(const uint32_t* __restrict__ w,
1596
+ uint32_t* __restrict__ w_new,
1597
+ const int* __restrict__ q_perm,
1598
+ const int w_width) {
1599
+ const uint64_t* w2 = (uint64_t*)w;
1600
+ uint64_t* w_new2 = (uint64_t*)w_new;
1601
+ int w2_stride = w_width >> 1;
1602
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1603
+ if (w2_column >= w2_stride) return;
1604
+ int w_new2_row = blockIdx.y;
1605
+ int q_perm_idx = w_new2_row << 3;
1606
+ uint64_t dst = 0;
1607
+
1608
+#pragma unroll
1609
+ for (int i = 0; i < 8; i++) {
1610
+ int source_row = q_perm[q_perm_idx++];
1611
+
1612
+ int w2_row = source_row >> 3;
1613
+ int w2_subrow = source_row & 0x07;
1614
+ int w2_row_shift = w2_subrow << 2;
1615
+ int wnew2_row_shift = i << 2;
1616
+
1617
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
1618
+ src >>= w2_row_shift;
1619
+ src &= 0x0000000f0000000f;
1620
+ src <<= wnew2_row_shift;
1621
+ dst |= src;
1622
+ }
1623
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
1624
}
1625
1802
-__global__ void make_sequential_2bit_kernel
1803
-(
1804
- const uint32_t* __restrict__ w,
1805
- uint32_t* __restrict__ w_new,
1806
- const int* __restrict__ q_perm,
1807
- const int w_width
1808
-)
1809
-{
1810
- const uint64_t* w2 = (uint64_t*) w;
1811
- uint64_t* w_new2 = (uint64_t*) w_new;
1812
- int w2_stride = w_width >> 1;
1813
- int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1814
- if (w2_column >= w2_stride) return;
1815
- int w_new2_row = blockIdx.y;
1816
- int q_perm_idx = w_new2_row << 4;
1817
- uint64_t dst = 0;
1818
-
1819
- #pragma unroll
1820
- for (int i = 0; i < 16; i++)
1821
- {
1822
- int source_row = q_perm[q_perm_idx++];
1823
-
1824
- int w2_row = source_row >> 4;
1825
- int w2_subrow = source_row & 0x0f;
1826
- int w2_row_shift = w2_subrow << 1;
1827
- int wnew2_row_shift = i << 1;
1828
-
1829
- uint64_t src = w2[w2_row * w2_stride + w2_column];
1830
- src >>= w2_row_shift;
1831
- src &= 0x0000000300000003;
1832
- src <<= wnew2_row_shift;
1833
- dst |= src;
1834
- }
1835
- w_new2[w_new2_row * w2_stride + w2_column] = dst;
1626
+__global__ void make_sequential_2bit_kernel(const uint32_t* __restrict__ w,
1627
+ uint32_t* __restrict__ w_new,
1628
+ const int* __restrict__ q_perm,
1629
+ const int w_width) {
1630
+ const uint64_t* w2 = (uint64_t*)w;
1631
+ uint64_t* w_new2 = (uint64_t*)w_new;
1632
+ int w2_stride = w_width >> 1;
1633
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1634
+ if (w2_column >= w2_stride) return;
1635
+ int w_new2_row = blockIdx.y;
1636
+ int q_perm_idx = w_new2_row << 4;
1637
+ uint64_t dst = 0;
1638
+
1639
+#pragma unroll
1640
+ for (int i = 0; i < 16; i++) {
1641
+ int source_row = q_perm[q_perm_idx++];
1642
+
1643
+ int w2_row = source_row >> 4;
1644
+ int w2_subrow = source_row & 0x0f;
1645
+ int w2_row_shift = w2_subrow << 1;
1646
+ int wnew2_row_shift = i << 1;
1647
+
1648
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
1649
+ src >>= w2_row_shift;
1650
+ src &= 0x0000000300000003;
1651
+ src <<= wnew2_row_shift;
1652
+ dst |= src;
1653
+ }
1654
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
1655
}
1656
1838
-__global__ void make_sequential_3bit_kernel
1839
-(
1840
- const uint32_t* __restrict__ w,
1841
- uint32_t* __restrict__ w_new,
1842
- const int* __restrict__ q_perm,
1843
- const int w_width
1844
-)
1845
-{
1846
- int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1847
- if (w_column >= w_width) return;
1848
- int w_new_row = blockIdx.y * 3;
1849
- int q_perm_idx = blockIdx.y << 5;
1850
- uint32_t dst[3] = {0, 0, 0};
1851
-
1852
- #pragma unroll
1853
- for (int i = 0; i < 32; i++)
1854
- {
1855
- int source_row = q_perm[q_perm_idx++];
1856
- int z_w = (source_row / 32) * 3;
1857
- int z_mod = source_row % 32;
1858
- int z_bit;
1859
-
1860
- if (z_mod != 10){
1861
- if (z_mod != 21){
1862
- z_bit = z_mod;
1863
- if (z_bit > 21){
1864
- z_bit *= 3;
1865
- z_bit -= 64;
1866
- z_w += 2;
1867
- } else if (z_bit > 10){
1868
- z_bit *= 3;
1869
- z_bit -= 32;
1870
- z_w += 1;
1871
- } else {
1872
- z_bit *= 3;
1873
- }
1874
- } else {
1875
- z_w += 1;
1876
- }
1877
- }
1878
-
1879
- uint64_t src;
1880
- if (z_mod == 10) {
1881
- src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1882
- } else if (z_mod == 21){
1883
- src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1657
+__global__ void make_sequential_3bit_kernel(const uint32_t* __restrict__ w,
1658
+ uint32_t* __restrict__ w_new,
1659
+ const int* __restrict__ q_perm,
1660
+ const int w_width) {
1661
+ int w_column = THREADS_X * blockIdx.x + threadIdx.x;
1662
+ if (w_column >= w_width) return;
1663
+ int w_new_row = blockIdx.y * 3;
1664
+ int q_perm_idx = blockIdx.y << 5;
1665
+ uint32_t dst[3] = {0, 0, 0};
1666
+
1667
+#pragma unroll
1668
+ for (int i = 0; i < 32; i++) {
1669
+ int source_row = q_perm[q_perm_idx++];
1670
+ int z_w = (source_row / 32) * 3;
1671
+ int z_mod = source_row % 32;
1672
+ int z_bit;
1673
+
1674
+ if (z_mod != 10) {
1675
+ if (z_mod != 21) {
1676
+ z_bit = z_mod;
1677
+ if (z_bit > 21) {
1678
+ z_bit *= 3;
1679
+ z_bit -= 64;
1680
+ z_w += 2;
1681
+ } else if (z_bit > 10) {
1682
+ z_bit *= 3;
1683
+ z_bit -= 32;
1684
+ z_w += 1;
1685
} else {
1885
- src = w[z_w * w_width + w_column];
1886
- src >>= z_bit;
1887
- src &= 0x07;
1686
+ z_bit *= 3;
1687
}
1688
+ } else {
1689
+ z_w += 1;
1690
+ }
1691
+ }
1692
1890
- z_w = 0;
1891
- if (i != 10){
1892
- if (i != 21){
1893
- z_bit = i;
1894
- if (z_bit > 21){
1895
- z_bit *= 3;
1896
- z_bit -= 64;
1897
- z_w += 2;
1898
- } else if (z_bit > 10){
1899
- z_bit *= 3;
1900
- z_bit -= 32;
1901
- z_w += 1;
1902
- } else {
1903
- z_bit *= 3;
1904
- }
1905
- } else {
1906
- z_w += 1;
1907
- }
1908
- }
1909
- if (i == 10) {
1910
- dst[z_w] |= (src & 0x03) << 30;
1911
- dst[z_w + 1] |= ((src & 0x4) >> 2);
1912
- } else if (i == 21) {
1913
- dst[z_w] |= (src & 0x01) << 31;
1914
- dst[z_w + 1] |= ((src & 0x6) >> 1);
1693
+ uint64_t src;
1694
+ if (z_mod == 10) {
1695
+ src = (w[z_w * w_width + w_column] >> 30) |
1696
+ ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
1697
+ } else if (z_mod == 21) {
1698
+ src = (w[z_w * w_width + w_column] >> 31) |
1699
+ ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
1700
+ } else {
1701
+ src = w[z_w * w_width + w_column];
1702
+ src >>= z_bit;
1703
+ src &= 0x07;
1704
+ }
1705
+
1706
+ z_w = 0;
1707
+ if (i != 10) {
1708
+ if (i != 21) {
1709
+ z_bit = i;
1710
+ if (z_bit > 21) {
1711
+ z_bit *= 3;
1712
+ z_bit -= 64;
1713
+ z_w += 2;
1714
+ } else if (z_bit > 10) {
1715
+ z_bit *= 3;
1716
+ z_bit -= 32;
1717
+ z_w += 1;
1718
} else {
1916
- dst[z_w] |= (src << z_bit);
1719
+ z_bit *= 3;
1720
}
1721
+ } else {
1722
+ z_w += 1;
1723
+ }
1724
+ }
1725
+ if (i == 10) {
1726
+ dst[z_w] |= (src & 0x03) << 30;
1727
+ dst[z_w + 1] |= ((src & 0x4) >> 2);
1728
+ } else if (i == 21) {
1729
+ dst[z_w] |= (src & 0x01) << 31;
1730
+ dst[z_w + 1] |= ((src & 0x6) >> 1);
1731
+ } else {
1732
+ dst[z_w] |= (src << z_bit);
1733
}
1919
- w_new[w_new_row * w_width + w_column] = dst[0];
1920
- w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1921
- w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1734
+ }
1735
+ w_new[w_new_row * w_width + w_column] = dst[0];
1736
+ w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
1737
+ w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
1738
}
1739
1924
-__global__ void make_sequential_8bit_kernel
1925
-(
1926
- const uint32_t* __restrict__ w,
1927
- uint32_t* __restrict__ w_new,
1928
- const int* __restrict__ q_perm,
1929
- const int w_width
1930
-)
1931
-{
1932
- const uint64_t* w2 = (uint64_t*) w;
1933
- uint64_t* w_new2 = (uint64_t*) w_new;
1934
- int w2_stride = w_width >> 1;
1935
- int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1936
- if (w2_column >= w2_stride) return;
1937
- int w_new2_row = blockIdx.y;
1938
- int q_perm_idx = w_new2_row << 2;
1939
- uint64_t dst = 0;
1940
-
1941
- #pragma unroll
1942
- for (int i = 0; i < 4; i++)
1943
- {
1944
- int source_row = q_perm[q_perm_idx++];
1945
-
1946
- int w2_row = source_row >> 2;
1947
- int w2_subrow = source_row & 0x03;
1948
- int w2_row_shift = w2_subrow << 3;
1949
- int wnew2_row_shift = i << 3;
1950
-
1951
- uint64_t src = w2[w2_row * w2_stride + w2_column];
1952
- src >>= w2_row_shift;
1953
- src &= 0x000000ff000000ff;
1954
- src <<= wnew2_row_shift;
1955
- dst |= src;
1956
- }
1957
- w_new2[w_new2_row * w2_stride + w2_column] = dst;
1740
+__global__ void make_sequential_8bit_kernel(const uint32_t* __restrict__ w,
1741
+ uint32_t* __restrict__ w_new,
1742
+ const int* __restrict__ q_perm,
1743
+ const int w_width) {
1744
+ const uint64_t* w2 = (uint64_t*)w;
1745
+ uint64_t* w_new2 = (uint64_t*)w_new;
1746
+ int w2_stride = w_width >> 1;
1747
+ int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
1748
+ if (w2_column >= w2_stride) return;
1749
+ int w_new2_row = blockIdx.y;
1750
+ int q_perm_idx = w_new2_row << 2;
1751
+ uint64_t dst = 0;
1752
+
1753
+#pragma unroll
1754
+ for (int i = 0; i < 4; i++) {
1755
+ int source_row = q_perm[q_perm_idx++];
1756
+
1757
+ int w2_row = source_row >> 2;
1758
+ int w2_subrow = source_row & 0x03;
1759
+ int w2_row_shift = w2_subrow << 3;
1760
+ int wnew2_row_shift = i << 3;
1761
+
1762
+ uint64_t src = w2[w2_row * w2_stride + w2_column];
1763
+ src >>= w2_row_shift;
1764
+ src &= 0x000000ff000000ff;
1765
+ src <<= wnew2_row_shift;
1766
+ dst |= src;
1767
+ }
1768
+ w_new2[w_new2_row * w2_stride + w2_column] = dst;
1769
}
1770
1771
+void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
1772
+ int width, int bit) {
1773
+ if (q_perm) {
1774
+ uint32_t* new_qweight = NULL;
1775
+ cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1776
1961
-void shuffle_exllama_weight
1962
-(
1963
- uint32_t* q_weight,
1964
- int* q_perm,
1965
- int height,
1966
- int width,
1967
- int bit
1968
-)
1969
-{
1970
- if (q_perm)
1971
- {
1972
- uint32_t* new_qweight = NULL;
1973
- cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
1974
-
1975
- dim3 blockDim, gridDim;
1976
- blockDim.x = THREADS_X;
1977
- blockDim.y = 1;
1978
- gridDim.x = DIVIDE(width, THREADS_X);
1979
- gridDim.y = height / 32 * bit;
1980
-
1981
- auto kernel = make_sequential_4bit_kernel;
1982
- if (bit == 2) {
1983
- kernel = make_sequential_2bit_kernel;
1984
- } else if (bit == 3) {
1985
- kernel = make_sequential_3bit_kernel;
1986
- gridDim.y = height / 32;
1987
- } else if (bit == 8) {
1988
- kernel = make_sequential_8bit_kernel;
1989
- }
1990
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1991
- kernel<<<gridDim, blockDim, 0, stream>>>
1992
- (
1993
- q_weight,
1994
- new_qweight,
1995
- q_perm,
1996
- width
1997
- );
1998
- // Replace qweights
1999
- cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
2000
- // Cleanup
2001
- cudaDeviceSynchronize();
2002
- cudaFree(new_qweight);
2003
- }
1777
dim3 blockDim, gridDim;
1778
blockDim.x = THREADS_X;
1779
blockDim.y = 1;
1780
gridDim.x = DIVIDE(width, THREADS_X);
2008
- gridDim.y = 1;
2009
- auto shuffle_kernel = shuffle_4bit_kernel;
1781
+ gridDim.y = height / 32 * bit;
1782
+
1783
+ auto kernel = make_sequential_4bit_kernel;
1784
if (bit == 2) {
2011
- shuffle_kernel = shuffle_2bit_kernel;
1785
+ kernel = make_sequential_2bit_kernel;
1786
} else if (bit == 3) {
2013
- shuffle_kernel = shuffle_3bit_kernel;
1787
+ kernel = make_sequential_3bit_kernel;
1788
+ gridDim.y = height / 32;
1789
} else if (bit == 8) {
2015
- shuffle_kernel = shuffle_8bit_kernel;
1790
+ kernel = make_sequential_8bit_kernel;
1791
}
1792
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
2018
- shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
1793
+ kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, new_qweight, q_perm,
1794
+ width);
1795
+ // Replace qweights
1796
+ cudaMemcpyAsync(q_weight, new_qweight,
1797
+ height / 32 * bit * width * sizeof(uint32_t),
1798
+ cudaMemcpyDeviceToDevice);
1799
+ // Cleanup
1800
+ cudaDeviceSynchronize();
1801
+ cudaFree(new_qweight);
1802
+ }
1803
+ dim3 blockDim, gridDim;
1804
+ blockDim.x = THREADS_X;
1805
+ blockDim.y = 1;
1806
+ gridDim.x = DIVIDE(width, THREADS_X);
1807
+ gridDim.y = 1;
1808
+ auto shuffle_kernel = shuffle_4bit_kernel;
1809
+ if (bit == 2) {
1810
+ shuffle_kernel = shuffle_2bit_kernel;
1811
+ } else if (bit == 3) {
1812
+ shuffle_kernel = shuffle_3bit_kernel;
1813
+ } else if (bit == 8) {
1814
+ shuffle_kernel = shuffle_8bit_kernel;
1815
+ }
1816
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
1817
+ shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
1818
}
1819
1820
} // namespace gptq
1821
} // namespace vllm
1822
2024
-torch::Tensor gptq_gemm
2025
-(
2026
- torch::Tensor a,
2027
- torch::Tensor b_q_weight,
2028
- torch::Tensor b_gptq_qzeros,
2029
- torch::Tensor b_gptq_scales,
2030
- torch::Tensor b_g_idx,
2031
- bool use_exllama,
2032
- int bit
2033
-)
2034
-{
2035
- const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
2036
- auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
2037
- at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
2038
- at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
2039
-
2040
- vllm::gptq::gemm_half_q_half_cuda
2041
- (
2042
- at::cuda::getCurrentCUDABlasHandle(),
2043
- (const half*) a.data_ptr(),
2044
- (const uint32_t*) b_q_weight.data_ptr(),
2045
- (const uint32_t*)b_gptq_qzeros.data_ptr(),
2046
- (const half*) b_gptq_scales.data_ptr(),
2047
- b_g_idx.device().is_meta() ? NULL : (const int*) b_g_idx.data_ptr(),
2048
- (half*) c.data_ptr(),
2049
- (half*) temp_dq.data_ptr(),
2050
- c.size(0), // m
2051
- c.size(1), // n
2052
- a.size(1), // k
2053
- b_gptq_qzeros.size(0), // group number
2054
- use_exllama,
2055
- bit
2056
- );
2057
- return c;
1823
+torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
1824
+ torch::Tensor b_gptq_qzeros,
1825
+ torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
1826
+ bool use_exllama, int bit) {
1827
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1828
+ auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1829
+ at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);
1830
+ at::Tensor temp_dq = torch::empty(
1831
+ {b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options);
1832
+
1833
+ vllm::gptq::gemm_half_q_half_cuda(
1834
+ at::cuda::getCurrentCUDABlasHandle(), (const half*)a.data_ptr(),
1835
+ (const uint32_t*)b_q_weight.data_ptr(),
1836
+ (const uint32_t*)b_gptq_qzeros.data_ptr(),
1837
+ (const half*)b_gptq_scales.data_ptr(),
1838
+ b_g_idx.device().is_meta() ? NULL : (const int*)b_g_idx.data_ptr(),
1839
+ (half*)c.data_ptr(), (half*)temp_dq.data_ptr(),
1840
+ c.size(0), // m
1841
+ c.size(1), // n
1842
+ a.size(1), // k
1843
+ b_gptq_qzeros.size(0), // group number
1844
+ use_exllama, bit);
1845
+ return c;
1846
}
1847
2060
-void gptq_shuffle
2061
-(
2062
- torch::Tensor q_weight,
2063
- torch::Tensor q_perm,
2064
- int bit
2065
-)
2066
-{
2067
- const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068
- vllm::gptq::shuffle_exllama_weight(
2069
- (uint32_t*) q_weight.data_ptr(),
2070
- q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*) q_perm.data_ptr(),
2071
- q_weight.size(0) * 32 / bit,
2072
- q_weight.size(1),
2073
- bit
2074
- );
1848
+void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int bit) {
1849
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
1850
+ vllm::gptq::shuffle_exllama_weight(
1851
+ (uint32_t*)q_weight.data_ptr(),
1852
+ q_perm.device().is_meta() || q_perm.numel() == 0
1853
+ ? NULL
1854
+ : (int*)q_perm.data_ptr(),
1855
+ q_weight.size(0) * 32 / bit, q_weight.size(1), bit);
1856
}
csrc/quantization/gptq/qdq_2.cuh
+48
-59
@@ -14,71 +14,60 @@ namespace gptq {
14
//
15
// ffddbb99 77553311 eeccaa88 66442200
16
17
-__forceinline__ __device__ void shuffle_2bit_16
18
-(
19
- uint32_t* q,
20
- int stride
21
-)
22
-{
23
- uint32_t qa = q[0];
24
- uint32_t qb = 0;
17
+__forceinline__ __device__ void shuffle_2bit_16(uint32_t* q, int stride) {
18
+ uint32_t qa = q[0];
19
+ uint32_t qb = 0;
20
26
- #pragma unroll
27
- for (int i = 0; i < 8; i++)
28
- {
29
- uint32_t qa0 = qa & 0x03;
30
- uint32_t qa1 = (qa & 0x0c) >> 2;
31
- qa >>= 4;
32
- qb |= (qa1 << (i * 2 + 16));
33
- qb |= (qa0 << (i * 2));
34
- }
35
- q[0] = qb;
21
+#pragma unroll
22
+ for (int i = 0; i < 8; i++) {
23
+ uint32_t qa0 = qa & 0x03;
24
+ uint32_t qa1 = (qa & 0x0c) >> 2;
25
+ qa >>= 4;
26
+ qb |= (qa1 << (i * 2 + 16));
27
+ qb |= (qa0 << (i * 2));
28
+ }
29
+ q[0] = qb;
30
}
31
38
-__forceinline__ __device__ void dequant_2bit_16
39
-(
40
- const uint32_t q_0,
41
- half2 (&dq)[8],
42
- int stride,
43
- const uint32_t zero
44
-)
45
-{
46
- const uint32_t c0 = 0x64006400;
47
- const half y4_ = __float2half_rn(1.0f / 4.0f);
48
- const half y16_ = __float2half_rn(1.0f / 16.0f);
49
- const half y64_ = __float2half_rn(1.0f / 64.0f);
50
- const half2 y4 = __halves2half2(y4_, y4_);
51
- const half2 y16 = __halves2half2(y16_, y16_);
52
- const half2 y64 = __halves2half2(y64_, y64_);
32
+__forceinline__ __device__ void dequant_2bit_16(const uint32_t q_0,
33
+ half2 (&dq)[8], int stride,
34
+ const uint32_t zero) {
35
+ const uint32_t c0 = 0x64006400;
36
+ const half y4_ = __float2half_rn(1.0f / 4.0f);
37
+ const half y16_ = __float2half_rn(1.0f / 16.0f);
38
+ const half y64_ = __float2half_rn(1.0f / 64.0f);
39
+ const half2 y4 = __halves2half2(y4_, y4_);
40
+ const half2 y16 = __halves2half2(y16_, y16_);
41
+ const half2 y64 = __halves2half2(y64_, y64_);
42
54
- const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
55
- const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
56
- const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
57
- const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
58
- const half2 z1 = __half2half2(z1_.as_half);
59
- const half2 z4 = __half2half2(z4_);
60
- const half2 z16 = __half2half2(z16_);
61
- const half2 z64 = __half2half2(z64_);
43
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
44
+ const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero));
45
+ const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
46
+ const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
47
+ const half2 z1 = __half2half2(z1_.as_half);
48
+ const half2 z4 = __half2half2(z4_);
49
+ const half2 z16 = __half2half2(z16_);
50
+ const half2 z64 = __half2half2(z64_);
51
63
- uint32_t qa = q_0;
64
- half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
65
- half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
66
- half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
67
- half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
68
- qa >>= 8;
69
- half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
70
- half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
71
- half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
72
- half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
52
+ uint32_t qa = q_0;
53
+ half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
54
+ half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
55
+ half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
56
+ half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
57
+ qa >>= 8;
58
+ half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
59
+ half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
60
+ half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
61
+ half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
62
74
- dq[0] = __hadd2(q0.as_half2, z1);
75
- dq[1] = __hfma2(q1.as_half2, y4, z4);
76
- dq[2] = __hfma2(q2.as_half2, y16, z16);
77
- dq[3] = __hfma2(q3.as_half2, y64, z64);
78
- dq[4] = __hadd2(q4.as_half2, z1);
79
- dq[5] = __hfma2(q5.as_half2, y4, z4);
80
- dq[6] = __hfma2(q6.as_half2, y16, z16);
81
- dq[7] = __hfma2(q7.as_half2, y64, z64);
63
+ dq[0] = __hadd2(q0.as_half2, z1);
64
+ dq[1] = __hfma2(q1.as_half2, y4, z4);
65
+ dq[2] = __hfma2(q2.as_half2, y16, z16);
66
+ dq[3] = __hfma2(q3.as_half2, y64, z64);
67
+ dq[4] = __hadd2(q4.as_half2, z1);
68
+ dq[5] = __hfma2(q5.as_half2, y4, z4);
69
+ dq[6] = __hfma2(q6.as_half2, y16, z16);
70
+ dq[7] = __hfma2(q7.as_half2, y64, z64);
71
}
72
73
} // namespace gptq
csrc/quantization/gptq/qdq_3.cuh
+127
-119
@@ -11,128 +11,136 @@ namespace gptq {
11
// vjjjhhhf ffdddbbb uiiiggge eecccaaa
12
// vtttrrrp ppnnnlll usssqqqo oommmkkk
13
14
-__forceinline__ __device__ void shuffle_3bit_32
15
-(
16
- uint32_t* q,
17
- int stride
18
-)
19
-{
20
- uint32_t qa = q[0 * stride];
21
- uint32_t qb = q[1 * stride];
22
- uint32_t qc = q[2 * stride];
23
-
24
- // qa: aa999888 77766655 54443332 22111000
25
- // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
26
- // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
27
-
28
- uint32_t qd = qc >> 26;
29
- qc <<= 4;
30
- qc |= qb >> 28;
31
- qb <<= 2;
32
- qb |= qa >> 30;
33
-
34
- // qa: ..999888 77766655 54443332 22111000
35
- // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
36
- // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
37
- // qd: vvvuuu
38
-
39
- uint32_t za = 0;
40
- uint32_t zb = 0;
41
- uint32_t zc = 0;
42
-
43
- for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }
44
- for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }
45
- for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }
46
-
47
- // za: 9997775 55333111 8886664 44222000
48
- // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
49
- // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
50
- // qd: vvvuuu
51
-
52
- za |= ((qd & 0x01) >> 0) << 15;
53
- zb |= ((qd & 0x02) >> 1) << 15;
54
- zc |= ((qd & 0x04) >> 2) << 15;
55
- za |= ((qd & 0x08) >> 3) << 31;
56
- zb |= ((qd & 0x10) >> 4) << 31;
57
- zc |= ((qd & 0x20) >> 5) << 31;
58
-
59
- // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
60
- // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
61
- // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
62
-
63
- q[0 * stride] = za;
64
- q[1 * stride] = zb;
65
- q[2 * stride] = zc;
66
-}
67
-
68
-__forceinline__ __device__ void dequant_3bit_32
69
-(
70
- const uint32_t q_0,
71
- const uint32_t q_1,
72
- const uint32_t q_2,
73
- half2 (&dq)[16],
74
- int stride,
75
- const uint32_t zero
76
-)
77
-{
78
- const uint32_t c0 = 0x64006400;
79
- const half y8_ = __float2half_rn(1.0f / 8.0f);
80
- const half y64_ = __float2half_rn(1.0f / 64.0f);
81
- const half2 y8 = __halves2half2(y8_, y8_);
82
- const half2 y64 = __halves2half2(y64_, y64_);
83
- const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
84
- const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
85
- const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
86
- const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
87
- const half2 z8 = __halves2half2(z8_, z8_);
88
- const half2 z64 = __halves2half2(z64_, z64_);
89
-
90
- uint32_t qa = q_0;
91
- uint32_t qb = q_1;
92
- uint32_t qc = q_2;
93
-
94
- half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
95
- half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
14
+__forceinline__ __device__ void shuffle_3bit_32(uint32_t* q, int stride) {
15
+ uint32_t qa = q[0 * stride];
16
+ uint32_t qb = q[1 * stride];
17
+ uint32_t qc = q[2 * stride];
18
+
19
+ // qa: aa999888 77766655 54443332 22111000
20
+ // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
21
+ // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
22
+
23
+ uint32_t qd = qc >> 26;
24
+ qc <<= 4;
25
+ qc |= qb >> 28;
26
+ qb <<= 2;
27
+ qb |= qa >> 30;
28
+
29
+ // qa: ..999888 77766655 54443332 22111000
30
+ // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
31
+ // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
32
+ // qd: vvvuuu
33
+
34
+ uint32_t za = 0;
35
+ uint32_t zb = 0;
36
+ uint32_t zc = 0;
37
+
38
+ for (int i = 0; i < 5; i++) {
39
+ uint32_t t0 = qa & 0x07;
40
+ uint32_t t1 = (qa & 0x38) >> 3;
41
qa >>= 6;
97
- half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
98
- half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
99
- half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
100
- qa >>= 9;
101
- qa &= 0x00010001;
102
- half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
103
- half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
42
+ za |= (t0 << (i * 3));
43
+ za |= (t1 << (i * 3 + 16));
44
+ }
45
+ for (int i = 0; i < 5; i++) {
46
+ uint32_t t0 = qb & 0x07;
47
+ uint32_t t1 = (qb & 0x38) >> 3;
48
qb >>= 6;
105
- half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
106
- half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
107
- half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
108
- qb >>= 8;
109
- qb &= 0x00020002;
110
- half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
111
- half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
49
+ zb |= (t0 << (i * 3));
50
+ zb |= (t1 << (i * 3 + 16));
51
+ }
52
+ for (int i = 0; i < 5; i++) {
53
+ uint32_t t0 = qc & 0x07;
54
+ uint32_t t1 = (qc & 0x38) >> 3;
55
qc >>= 6;
113
- half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
114
- half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
115
- half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
116
- qc >>= 7;
117
- qc &= 0x00040004;
118
- half2_uint32 q15((qa | qb | qc) | c0);
119
-
120
- dq[ 0] = __hadd2( q0.as_half2, z1);
121
- dq[ 1] = __hfma2( q1.as_half2, y8, z8);
122
- dq[ 2] = __hadd2( q2.as_half2, z1);
123
- dq[ 3] = __hfma2( q3.as_half2, y8, z8);
124
- dq[ 4] = __hfma2( q4.as_half2, y64, z64);
125
- dq[ 5] = __hadd2( q5.as_half2, z1);
126
- dq[ 6] = __hfma2( q6.as_half2, y8, z8);
127
- dq[ 7] = __hadd2( q7.as_half2, z1);
128
- dq[ 8] = __hfma2( q8.as_half2, y8, z8);
129
- dq[ 9] = __hfma2( q9.as_half2, y64, z64);
130
- dq[10] = __hadd2(q10.as_half2, z1);
131
- dq[11] = __hfma2(q11.as_half2, y8, z8);
132
- dq[12] = __hadd2(q12.as_half2, z1);
133
- dq[13] = __hfma2(q13.as_half2, y8, z8);
134
- dq[14] = __hfma2(q14.as_half2, y64, z64);
135
- dq[15] = __hadd2(q15.as_half2, z1);
56
+ zc |= (t0 << (i * 3));
57
+ zc |= (t1 << (i * 3 + 16));
58
+ }
59
+
60
+ // za: 9997775 55333111 8886664 44222000
61
+ // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
62
+ // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
63
+ // qd: vvvuuu
64
+
65
+ za |= ((qd & 0x01) >> 0) << 15;
66
+ zb |= ((qd & 0x02) >> 1) << 15;
67
+ zc |= ((qd & 0x04) >> 2) << 15;
68
+ za |= ((qd & 0x08) >> 3) << 31;
69
+ zb |= ((qd & 0x10) >> 4) << 31;
70
+ zc |= ((qd & 0x20) >> 5) << 31;
71
+
72
+ // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
73
+ // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
74
+ // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
75
+
76
+ q[0 * stride] = za;
77
+ q[1 * stride] = zb;
78
+ q[2 * stride] = zc;
79
+}
80
+
81
+__forceinline__ __device__ void dequant_3bit_32(const uint32_t q_0,
82
+ const uint32_t q_1,
83
+ const uint32_t q_2,
84
+ half2 (&dq)[16], int stride,
85
+ const uint32_t zero) {
86
+ const uint32_t c0 = 0x64006400;
87
+ const half y8_ = __float2half_rn(1.0f / 8.0f);
88
+ const half y64_ = __float2half_rn(1.0f / 64.0f);
89
+ const half2 y8 = __halves2half2(y8_, y8_);
90
+ const half2 y64 = __halves2half2(y64_, y64_);
91
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
92
+ const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero));
93
+ const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero));
94
+ const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half);
95
+ const half2 z8 = __halves2half2(z8_, z8_);
96
+ const half2 z64 = __halves2half2(z64_, z64_);
97
+
98
+ uint32_t qa = q_0;
99
+ uint32_t qb = q_1;
100
+ uint32_t qc = q_2;
101
+
102
+ half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
103
+ half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
104
+ qa >>= 6;
105
+ half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
106
+ half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
107
+ half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
108
+ qa >>= 9;
109
+ qa &= 0x00010001;
110
+ half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
111
+ half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
112
+ qb >>= 6;
113
+ half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
114
+ half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
115
+ half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
116
+ qb >>= 8;
117
+ qb &= 0x00020002;
118
+ half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
119
+ half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
120
+ qc >>= 6;
121
+ half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
122
+ half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
123
+ half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
124
+ qc >>= 7;
125
+ qc &= 0x00040004;
126
+ half2_uint32 q15((qa | qb | qc) | c0);
127
+
128
+ dq[0] = __hadd2(q0.as_half2, z1);
129
+ dq[1] = __hfma2(q1.as_half2, y8, z8);
130
+ dq[2] = __hadd2(q2.as_half2, z1);
131
+ dq[3] = __hfma2(q3.as_half2, y8, z8);
132
+ dq[4] = __hfma2(q4.as_half2, y64, z64);
133
+ dq[5] = __hadd2(q5.as_half2, z1);
134
+ dq[6] = __hfma2(q6.as_half2, y8, z8);
135
+ dq[7] = __hadd2(q7.as_half2, z1);
136
+ dq[8] = __hfma2(q8.as_half2, y8, z8);
137
+ dq[9] = __hfma2(q9.as_half2, y64, z64);
138
+ dq[10] = __hadd2(q10.as_half2, z1);
139
+ dq[11] = __hfma2(q11.as_half2, y8, z8);
140
+ dq[12] = __hadd2(q12.as_half2, z1);
141
+ dq[13] = __hfma2(q13.as_half2, y8, z8);
142
+ dq[14] = __hfma2(q14.as_half2, y64, z64);
143
+ dq[15] = __hadd2(q15.as_half2, z1);
144
}
145
146
} // namespace gptq
csrc/quantization/gptq/qdq_4.cuh
+91
-112
@@ -13,133 +13,112 @@ namespace gptq {
13
//
14
// 77775555 33331111 66664444 22220000
15
16
-__forceinline__ __device__ void shuffle_4bit_8
17
-(
18
- uint32_t* q,
19
- int stride
20
-)
21
-{
22
- uint32_t qa = q[0];
23
- uint32_t qb = 0;
24
-
25
- #pragma unroll
26
- for (int i = 0; i < 4; i++)
27
- {
28
- uint32_t qa0 = qa & 0x0f;
29
- uint32_t qa1 = (qa & 0xf0) >> 4;
30
- qa >>= 8;
31
- qb |= (qa1 << (i * 4 + 16));
32
- qb |= (qa0 << (i * 4));
33
- }
34
- q[0] = qb;
35
-}
36
-
37
-__forceinline__ __device__ void dequant_4bit_8
38
-(
39
- const uint32_t q_0,
40
- half2 (&dq)[4],
41
- int stride,
42
- const uint32_t zero
43
-)
44
-{
45
- const uint32_t c0 = 0x64006400;
46
- const half y16_ = __float2half_rn(1.0f / 16.0f);
47
- const half2 y16 = __halves2half2(y16_, y16_);
48
- const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
49
- const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
50
- const half2 z1 = __half2half2(z1_.as_half);
51
- const half2 z16 = __half2half2(z16_);
52
-
53
- uint32_t qa = q_0;
54
- half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
55
- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
16
+__forceinline__ __device__ void shuffle_4bit_8(uint32_t* q, int stride) {
17
+ uint32_t qa = q[0];
18
+ uint32_t qb = 0;
19
+
20
+#pragma unroll
21
+ for (int i = 0; i < 4; i++) {
22
+ uint32_t qa0 = qa & 0x0f;
23
+ uint32_t qa1 = (qa & 0xf0) >> 4;
24
qa >>= 8;
57
- half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
58
- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
25
+ qb |= (qa1 << (i * 4 + 16));
26
+ qb |= (qa0 << (i * 4));
27
+ }
28
+ q[0] = qb;
29
+}
30
60
- dq[0] = __hadd2(q0.as_half2, z1);
61
- dq[1] = __hfma2(q1.as_half2, y16, z16);
62
- dq[2] = __hadd2(q2.as_half2, z1);
63
- dq[3] = __hfma2(q3.as_half2, y16, z16);
31
+__forceinline__ __device__ void dequant_4bit_8(const uint32_t q_0,
32
+ half2 (&dq)[4], int stride,
33
+ const uint32_t zero) {
34
+ const uint32_t c0 = 0x64006400;
35
+ const half y16_ = __float2half_rn(1.0f / 16.0f);
36
+ const half2 y16 = __halves2half2(y16_, y16_);
37
+ const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero);
38
+ const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero));
39
+ const half2 z1 = __half2half2(z1_.as_half);
40
+ const half2 z16 = __half2half2(z16_);
41
+
42
+ uint32_t qa = q_0;
43
+ half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
44
+ half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
45
+ qa >>= 8;
46
+ half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
47
+ half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
48
+
49
+ dq[0] = __hadd2(q0.as_half2, z1);
50
+ dq[1] = __hfma2(q1.as_half2, y16, z16);
51
+ dq[2] = __hadd2(q2.as_half2, z1);
52
+ dq[3] = __hfma2(q3.as_half2, y16, z16);
53
}
54
66
-__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
67
-(
68
- const uint32_t zero,
69
- const half scale,
70
- half2 (&z1z16)[2],
71
- half2 (&y1y16)[2]
72
-)
73
-{
74
- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
75
- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
55
+__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale(
56
+ const uint32_t zero, const half scale, half2 (&z1z16)[2],
57
+ half2 (&y1y16)[2]) {
58
+ half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
59
+ half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
60
77
- half2 scale2 = __half2half2(scale);
61
+ half2 scale2 = __half2half2(scale);
62
79
- z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
80
- z1z16[1] = __hmul2(scale2, __half2half2(z16));
63
+ z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
64
+ z1z16[1] = __hmul2(scale2, __half2half2(z16));
65
82
- const half y1 = __float2half_rn(1.0f);
83
- const half y16 = __float2half_rn(1.0f / 16.0f);
66
+ const half y1 = __float2half_rn(1.0f);
67
+ const half y16 = __float2half_rn(1.0f / 16.0f);
68
85
- y1y16[0] = __hmul2(scale2, __half2half2(y1));
86
- y1y16[1] = __hmul2(scale2, __half2half2(y16));
69
+ y1y16[0] = __hmul2(scale2, __half2half2(y1));
70
+ y1y16[1] = __hmul2(scale2, __half2half2(y16));
71
}
72
89
-__forceinline__ __device__ void dequant_4bit_8_prep_zero
90
-(
91
- const uint32_t zero,
92
- half2(&z1z16)[2],
93
- half2(&y1y16)[2]
94
-)
95
-{
96
- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
97
- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
73
+__forceinline__ __device__ void dequant_4bit_8_prep_zero(const uint32_t zero,
74
+ half2 (&z1z16)[2],
75
+ half2 (&y1y16)[2]) {
76
+ half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
77
+ half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
78
99
- z1z16[0] = __half2half2(z1.as_half);
100
- z1z16[1] = __half2half2(z16);
79
+ z1z16[0] = __half2half2(z1.as_half);
80
+ z1z16[1] = __half2half2(z16);
81
102
- const half y1 = __float2half_rn(1.0f);
103
- const half y16 = __float2half_rn(1.0f / 16.0f);
82
+ const half y1 = __float2half_rn(1.0f);
83
+ const half y16 = __float2half_rn(1.0f / 16.0f);
84
105
- y1y16[0] = __half2half2(y1);
106
- y1y16[1] = __half2half2(y16);
85
+ y1y16[0] = __half2half2(y1);
86
+ y1y16[1] = __half2half2(y16);
87
}
88
109
-
110
-__forceinline__ __device__ void dequant_4bit_8_gptq
111
-(
112
- const uint32_t q_0,
113
- half2 (&dq)[4],
114
- half2 (&z1z16)[2],
115
- half2 (&y1y16)[2],
116
- int stride,
117
- bool scaled
118
-)
119
-{
120
- const uint32_t c0 = 0x64006400;
121
-
122
- uint32_t qa = q_0;
123
- half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )
124
- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
125
- qa >>= 8;
126
- half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )
127
- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
128
-
129
- if (scaled)
130
- {
131
- dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
132
- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
133
- dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
134
- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
135
- }
136
- else
137
- {
138
- dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
139
- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )
140
- dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
141
- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )
142
- }
89
+__forceinline__ __device__ void dequant_4bit_8_gptq(const uint32_t q_0,
90
+ half2 (&dq)[4],
91
+ half2 (&z1z16)[2],
92
+ half2 (&y1y16)[2],
93
+ int stride, bool scaled) {
94
+ const uint32_t c0 = 0x64006400;
95
+
96
+ uint32_t qa = q_0;
97
+ half2_uint32 q0((qa & 0x000f000f) |
98
+ c0); // half2( q[0] + 1024, q[1] + 1024 )
99
+ half2_uint32 q1((qa & 0x00f000f0) |
100
+ c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
101
+ qa >>= 8;
102
+ half2_uint32 q2((qa & 0x000f000f) |
103
+ c0); // half2( q[4] + 1024, q[5] + 1024 )
104
+ half2_uint32 q3((qa & 0x00f000f0) |
105
+ c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
106
+
107
+ if (scaled) {
108
+ dq[0] = __hfma2(q0.as_half2, y1y16[0],
109
+ z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
110
+ dq[1] = __hfma2(q1.as_half2, y1y16[1],
111
+ z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
112
+ dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
113
+ dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
114
+ } else {
115
+ dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
116
+ dq[1] = __hfma2(q1.as_half2, y1y16[1],
117
+ z1z16[1]); // half2( q[2] - z, q[3] - z )
118
+ dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
119
+ dq[3] = __hfma2(q3.as_half2, y1y16[1],
120
+ z1z16[1]); // half2( q[6] - z, q[7] - z )
121
+ }
122
}
123
} // namespace gptq
124
} // namespace vllm
csrc/quantization/gptq/qdq_8.cuh
+12
-22
@@ -10,28 +10,18 @@ Copied from https://github.com/turboderp/exllamav2
10
namespace vllm {
11
namespace gptq {
12
13
-__forceinline__ __device__ void shuffle_8bit_4
14
-(
15
- uint32_t* q,
16
- int stride
17
-)
18
-{
19
-}
20
-
21
-__forceinline__ __device__ void dequant_8bit_8
22
-(
23
- const uint32_t q_0,
24
- const uint32_t q_1,
25
- half2 (&dq)[4],
26
- int stride,
27
- const uint32_t zero
28
-)
29
-{
30
- half dqh[8];
31
- for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), zero);
32
- for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
33
-
34
- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
13
+__forceinline__ __device__ void shuffle_8bit_4(uint32_t* q, int stride) {}
14
+
15
+__forceinline__ __device__ void dequant_8bit_8(const uint32_t q_0,
16
+ const uint32_t q_1,
17
+ half2 (&dq)[4], int stride,
18
+ const uint32_t zero) {
19
+ half dqh[8];
20
+ for (int i = 0; i < 4; i++) dqh[i] = dq_ns(exb(q_0, i * 8, 0xff), zero);
21
+ for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero);
22
+
23
+ for (int i = 0; i < 4; i++)
24
+ dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
25
}
26
27
} // namespace gptq
csrc/quantization/gptq/qdq_util.cuh
+27
-31
@@ -8,51 +8,47 @@ Copied from https://github.com/turboderp/exllamav2
8
namespace vllm {
9
namespace gptq {
10
11
-union half2_uint32
12
-{
13
- uint32_t as_uint32;
14
- half2 as_half2;
15
- __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
16
- __device__ half2_uint32(half2 val) : as_half2(val) {}
11
+union half2_uint32 {
12
+ uint32_t as_uint32;
13
+ half2 as_half2;
14
+ __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
15
+ __device__ half2_uint32(half2 val) : as_half2(val) {}
16
};
17
19
-union half_uint16
20
-{
21
- uint16_t as_uint16;
22
- half as_half;
23
- __device__ half_uint16(uint16_t val) : as_uint16(val) {}
24
- __device__ half_uint16(half val) : as_half(val) {}
18
+union half_uint16 {
19
+ uint16_t as_uint16;
20
+ half as_half;
21
+ __device__ half_uint16(uint16_t val) : as_uint16(val) {}
22
+ __device__ half_uint16(half val) : as_half(val) {}
23
};
24
25
// Max_scale premultiplied by 1/256
26
29
-__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)
30
-{
31
- int qs_i = qs + 1;
32
- half qs_h = __int2half_rn(qs_i * qs_i);
33
- qs_h = __hmul(qs_h, max_scale);
34
- return qs_h;
27
+__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) {
28
+ int qs_i = qs + 1;
29
+ half qs_h = __int2half_rn(qs_i * qs_i);
30
+ qs_h = __hmul(qs_h, max_scale);
31
+ return qs_h;
32
}
33
37
-__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)
38
-{
39
- return __hmul(__int2half_rn(q - qzero), scale);
34
+__forceinline__ __device__ half dq(const int q, const int qzero,
35
+ const half scale) {
36
+ return __hmul(__int2half_rn(q - qzero), scale);
37
}
38
42
-__forceinline__ __device__ half dq_ns(const int q, const int qzero)
43
-{
44
- //return __hsub(__int2half_rn(q), __int2half_rn(qzero));
45
- return __int2half_rn(q - qzero);
39
+__forceinline__ __device__ half dq_ns(const int q, const int qzero) {
40
+ // return __hsub(__int2half_rn(q), __int2half_rn(qzero));
41
+ return __int2half_rn(q - qzero);
42
}
43
48
-__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)
49
-{
50
- return (int)((q >> shift) & mask);
44
+__forceinline__ __device__ int exb(const uint32_t q, const int shift,
45
+ const int mask) {
46
+ return (int)((q >> shift) & mask);
47
}
48
53
-__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)
54
-{
55
- return (int)(__funnelshift_rc(q0, q1, shift) & mask);
49
+__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0,
50
+ const int shift, const int mask) {
51
+ return (int)(__funnelshift_rc(q0, q1, shift) & mask);
52
}
53
54
} // namespace gptq
csrc/quantization/gptq_marlin/gptq_marlin.cu
+360
-336
@@ -22,53 +22,58 @@
22
#include "gptq_marlin.cuh"
23
#include "gptq_marlin_dtypes.cuh"
24
25
-#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) static_assert(\
26
- std::is_same<scalar_t, half>::value || std::is_same<scalar_t, nv_bfloat16>::value, \
27
- "only float16 and bfloat16 is supported");
28
-
29
-template <typename T> inline std::string str(T x) { return std::to_string(x); }
25
+#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
26
+ static_assert(std::is_same<scalar_t, half>::value || \
27
+ std::is_same<scalar_t, nv_bfloat16>::value, \
28
+ "only float16 and bfloat16 is supported");
29
+
30
+template <typename T>
31
+inline std::string str(T x) {
32
+ return std::to_string(x);
33
+}
34
35
namespace gptq_marlin {
36
37
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38
35
-__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
36
- int const *__restrict__ perm_int_ptr,
37
- int4 *__restrict__ out_int4_ptr, int size_m,
39
+__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
40
+ int const* __restrict__ perm_int_ptr,
41
+ int4* __restrict__ out_int4_ptr, int size_m,
42
int size_k, int block_rows) {}
43
40
-template <typename scalar_t, // compute dtype, half or nv_float16
41
- const int num_bits, // number of bits used for weights
42
- const int threads, // number of threads in a threadblock
43
- const int thread_m_blocks, // number of 16x16 blocks in the m
44
- // dimension (batchsize) of the threadblock
45
- const int thread_n_blocks, // same for n dimension (output)
46
- const int thread_k_blocks, // same for k dimension (reduction)
47
- const int stages, // number of stages for the async global->shared
48
- // fetch pipeline
49
- const bool has_act_order, // whether act_order is enabled
50
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
51
- // a separate quantization scale
44
+template <typename scalar_t, // compute dtype, half or nv_float16
45
+ const int num_bits, // number of bits used for weights
46
+ const int threads, // number of threads in a threadblock
47
+ const int thread_m_blocks, // number of 16x16 blocks in the m
48
+ // dimension (batchsize) of the
49
+ // threadblock
50
+ const int thread_n_blocks, // same for n dimension (output)
51
+ const int thread_k_blocks, // same for k dimension (reduction)
52
+ const int stages, // number of stages for the async global->shared
53
+ // fetch pipeline
54
+ const bool has_act_order, // whether act_order is enabled
55
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
56
+ // with a separate quantization scale
57
>
53
-__global__ void
54
-Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
55
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
56
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
57
- const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
58
- // (k/groupsize)xn
59
- const int *__restrict__ g_idx, // int32 group indices of shape k
60
- int num_groups, // number of scale groups per output channel
61
- int prob_m, // batch dimension m
62
- int prob_n, // output dimension n
63
- int prob_k, // reduction dimension k
64
- int *locks // extra global storage for barrier synchronization
58
+__global__ void Marlin(
59
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
60
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
61
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
62
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
63
+ // (k/groupsize)xn
64
+ const int* __restrict__ g_idx, // int32 group indices of shape k
65
+ int num_groups, // number of scale groups per output channel
66
+ int prob_m, // batch dimension m
67
+ int prob_n, // output dimension n
68
+ int prob_k, // reduction dimension k
69
+ int* locks // extra global storage for barrier synchronization
70
) {}
71
67
-} // namespace gptq_marlin
72
+} // namespace gptq_marlin
73
69
-torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
70
- torch::Tensor &b_scales, torch::Tensor &g_idx,
71
- torch::Tensor &perm, torch::Tensor &workspace,
74
+torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
75
+ torch::Tensor& b_scales, torch::Tensor& g_idx,
76
+ torch::Tensor& perm, torch::Tensor& workspace,
77
int64_t num_bits, int64_t size_m, int64_t size_n,
78
int64_t size_k, bool is_k_full) {
79
TORCH_CHECK_NOT_IMPLEMENTED(false,
@@ -81,24 +86,26 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
86
// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
87
// output/accumulation.
88
template <typename scalar_t>
84
-__device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
85
- const typename ScalarType<scalar_t>::FragB &frag_b,
86
- typename ScalarType<scalar_t>::FragC &frag_c) {
87
- const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
88
- const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
89
- float *c = reinterpret_cast<float *>(&frag_c);
89
+__device__ inline void mma(const typename ScalarType<scalar_t>::FragA& a_frag,
90
+ const typename ScalarType<scalar_t>::FragB& frag_b,
91
+ typename ScalarType<scalar_t>::FragC& frag_c) {
92
+ const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
93
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
94
+ float* c = reinterpret_cast<float*>(&frag_c);
95
if constexpr (std::is_same<scalar_t, half>::value) {
91
- asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
92
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
93
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
94
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
95
- "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
96
+ asm volatile(
97
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
98
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
101
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
102
} else if constexpr (std::is_same<scalar_t, nv_bfloat16>::value) {
97
- asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
98
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
99
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
100
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
101
- "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
103
+ asm volatile(
104
+ "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
105
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
106
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
107
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
108
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
109
} else {
110
STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
111
}
@@ -107,8 +114,9 @@ __device__ inline void mma(const typename ScalarType<scalar_t>::FragA &a_frag,
114
// Instruction for loading a full 16x16 matrix fragment of operand A from shared
115
// memory, directly in tensor core layout.
116
template <typename scalar_t>
110
-__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const void *smem_ptr) {
111
- uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
117
+__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA& frag_a,
118
+ const void* smem_ptr) {
119
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
120
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
121
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
122
: "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
@@ -118,7 +126,8 @@ __device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA &frag_a, const
126
// Lookup-table based 3-input logical operation; explicitly used for
127
// dequantization as the compiler does not seem to automatically recognize it in
128
// all cases.
121
-template <int lut> __device__ inline int lop3(int a, int b, int c) {
129
+template <int lut>
130
+__device__ inline int lop3(int a, int b, int c) {
131
int res;
132
asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
133
: "=r"(res)
@@ -140,8 +149,10 @@ __device__ inline uint32_t prmt(uint32_t a) {
149
// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
150
// values. We mostly follow the strategy in the link below, with some small
151
// changes:
143
-// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
144
-// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
152
+// - FP16:
153
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154
+// - BF16:
155
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385
156
template <typename scalar_t>
157
__device__ inline typename ScalarType<scalar_t>::FragB dequant_4bit(int q) {
158
STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
@@ -161,16 +172,17 @@ __device__ inline typename ScalarType<half>::FragB dequant_4bit<half>(int q) {
172
const int MUL = 0x2c002c00;
173
const int ADD = 0xd480d480;
174
typename ScalarType<half>::FragB frag_b;
164
- frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
165
- *reinterpret_cast<const half2 *>(&SUB));
166
- frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
167
- *reinterpret_cast<const half2 *>(&MUL),
168
- *reinterpret_cast<const half2 *>(&ADD));
175
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
176
+ *reinterpret_cast<const half2*>(&SUB));
177
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
178
+ *reinterpret_cast<const half2*>(&MUL),
179
+ *reinterpret_cast<const half2*>(&ADD));
180
return frag_b;
181
}
182
183
template <>
173
-__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat16>(int q) {
184
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
185
+dequant_4bit<nv_bfloat16>(int q) {
186
static constexpr uint32_t MASK = 0x000f000f;
187
static constexpr uint32_t EX = 0x43004300;
188
@@ -184,7 +196,7 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat
196
static constexpr uint32_t MUL = 0x3F803F80;
197
static constexpr uint32_t ADD = 0xC308C308;
198
187
- frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162 *>(&lo),
199
+ frag_b[0] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&lo),
200
*reinterpret_cast<const nv_bfloat162*>(&MUL),
201
*reinterpret_cast<const nv_bfloat162*>(&ADD));
202
frag_b[1] = __hfma2(*reinterpret_cast<nv_bfloat162*>(&hi),
@@ -193,10 +205,12 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_4bit<nv_bfloat
205
return frag_b;
206
}
207
196
-// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or bf16
197
-// Reference:
198
-// - FP16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
199
-// - BF16: https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
208
+// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or
209
+// bf16 Reference:
210
+// - FP16:
211
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
212
+// - BF16:
213
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
214
template <typename scalar_t>
215
__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
216
STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
@@ -214,24 +228,26 @@ __device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
228
static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
229
230
typename ScalarType<half>::FragB frag_b;
217
- frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
218
- *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
219
- frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
220
- *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
231
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
232
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
233
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
234
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
235
return frag_b;
236
}
237
238
template <>
225
-__device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat16>(int q) {
239
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
240
+dequant_8bit<nv_bfloat16>(int q) {
241
typename ScalarType<nv_bfloat16>::FragB frag_b;
242
243
float fp32_intermediates[4];
229
- uint32_t * fp32_intermediates_casted = reinterpret_cast<uint32_t*>(fp32_intermediates);
244
+ uint32_t* fp32_intermediates_casted =
245
+ reinterpret_cast<uint32_t*>(fp32_intermediates);
246
247
static constexpr uint32_t fp32_base = 0x4B000000;
232
- fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
248
+ fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650);
249
fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652);
234
- fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
250
+ fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651);
251
fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653);
252
253
fp32_intermediates[0] -= 8388736.f;
@@ -240,8 +256,10 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat
256
fp32_intermediates[3] -= 8388736.f;
257
258
uint32_t* bf16_result_ptr = reinterpret_cast<uint32_t*>(&frag_b);
243
- bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632);
244
- bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632);
259
+ bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0],
260
+ fp32_intermediates_casted[1], 0x7632);
261
+ bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2],
262
+ fp32_intermediates_casted[3], 0x7632);
263
264
return frag_b;
265
}
@@ -249,30 +267,32 @@ __device__ inline typename ScalarType<nv_bfloat16>::FragB dequant_8bit<nv_bfloat
267
// Multiply dequantized values by the corresponding quantization scale; used
268
// only for grouped quantization.
269
template <typename scalar_t>
252
-__device__ inline void scale(typename ScalarType<scalar_t>::FragB &frag_b,
253
- typename ScalarType<scalar_t>::FragS &frag_s, int i) {
270
+__device__ inline void scale(typename ScalarType<scalar_t>::FragB& frag_b,
271
+ typename ScalarType<scalar_t>::FragS& frag_s,
272
+ int i) {
273
using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
255
- scalar_t2 s = ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t *>(&frag_s)[i]);
274
+ scalar_t2 s =
275
+ ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t*>(&frag_s)[i]);
276
frag_b[0] = __hmul2(frag_b[0], s);
277
frag_b[1] = __hmul2(frag_b[1], s);
278
}
279
280
// Same as above, but for act_order (each K is multiplied individually)
281
template <typename scalar_t>
262
-__device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
263
- typename ScalarType<scalar_t>::FragS &frag_s_1,
264
- typename ScalarType<scalar_t>::FragS &frag_s_2,
265
- typename ScalarType<scalar_t>::FragS &frag_s_3,
266
- typename ScalarType<scalar_t>::FragS &frag_s_4,
282
+__device__ inline void scale4(typename ScalarType<scalar_t>::FragB& frag_b,
283
+ typename ScalarType<scalar_t>::FragS& frag_s_1,
284
+ typename ScalarType<scalar_t>::FragS& frag_s_2,
285
+ typename ScalarType<scalar_t>::FragS& frag_s_3,
286
+ typename ScalarType<scalar_t>::FragS& frag_s_4,
287
int i) {
268
- using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
288
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
289
scalar_t2 s_val_1_2;
270
- s_val_1_2.x = reinterpret_cast<scalar_t *>(&frag_s_1)[i];
271
- s_val_1_2.y = reinterpret_cast<scalar_t *>(&frag_s_2)[i];
290
+ s_val_1_2.x = reinterpret_cast<scalar_t*>(&frag_s_1)[i];
291
+ s_val_1_2.y = reinterpret_cast<scalar_t*>(&frag_s_2)[i];
292
293
scalar_t2 s_val_3_4;
274
- s_val_3_4.x = reinterpret_cast<scalar_t *>(&frag_s_3)[i];
275
- s_val_3_4.y = reinterpret_cast<scalar_t *>(&frag_s_4)[i];
294
+ s_val_3_4.x = reinterpret_cast<scalar_t*>(&frag_s_3)[i];
295
+ s_val_3_4.y = reinterpret_cast<scalar_t*>(&frag_s_4)[i];
296
297
frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
298
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
@@ -280,14 +300,15 @@ __device__ inline void scale4(typename ScalarType<scalar_t>::FragB &frag_b,
300
301
// Given 2 floats multiply by 2 scales (halves)
302
template <typename scalar_t>
283
-__device__ inline void scale_float(float *c, typename ScalarType<scalar_t>::FragS &s) {
284
- scalar_t *s_ptr = reinterpret_cast<scalar_t *>(&s);
303
+__device__ inline void scale_float(float* c,
304
+ typename ScalarType<scalar_t>::FragS& s) {
305
+ scalar_t* s_ptr = reinterpret_cast<scalar_t*>(&s);
306
c[0] = __fmul_rn(c[0], ScalarType<scalar_t>::num2float(s_ptr[0]));
307
c[1] = __fmul_rn(c[1], ScalarType<scalar_t>::num2float(s_ptr[1]));
308
}
309
310
// Wait until barrier reaches `count`, then lock for current threadblock.
290
-__device__ inline void barrier_acquire(int *lock, int count) {
311
+__device__ inline void barrier_acquire(int* lock, int count) {
312
if (threadIdx.x == 0) {
313
int state = -1;
314
do
@@ -302,7 +323,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
323
}
324
325
// Release barrier and increment visitation count.
305
-__device__ inline void barrier_release(int *lock, bool reset = false) {
326
+__device__ inline void barrier_release(int* lock, bool reset = false) {
327
__syncthreads();
328
if (threadIdx.x == 0) {
329
if (reset) {
@@ -321,11 +342,10 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
342
343
// For a given "a" of size [M,K] performs a permutation of the K columns based
344
// on the given "perm" indices.
324
-__global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
325
- int const *__restrict__ perm_int_ptr,
326
- int4 *__restrict__ out_int4_ptr, int size_m,
345
+__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
346
+ int const* __restrict__ perm_int_ptr,
347
+ int4* __restrict__ out_int4_ptr, int size_m,
348
int size_k, int block_rows) {
328
-
349
int start_row = block_rows * blockIdx.x;
350
int finish_row = start_row + block_rows;
351
if (finish_row > size_m) {
@@ -341,9 +361,8 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
361
362
int offset = row * row_stride;
363
344
- half const *a_row_half =
345
- reinterpret_cast<half const *>(a_int4_ptr + offset);
346
- half *out_half = reinterpret_cast<half *>(out_int4_ptr + offset);
364
+ half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
365
+ half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
366
367
int base_k = 0;
368
@@ -374,31 +393,32 @@ __global__ void permute_cols_kernel(int4 const *__restrict__ a_int4_ptr,
393
}
394
}
395
377
-template <typename scalar_t, // compute dtype, half or nv_float16
378
- const int num_bits, // number of bits used for weights
379
- const int threads, // number of threads in a threadblock
380
- const int thread_m_blocks, // number of 16x16 blocks in the m
381
- // dimension (batchsize) of the threadblock
382
- const int thread_n_blocks, // same for n dimension (output)
383
- const int thread_k_blocks, // same for k dimension (reduction)
384
- const int stages, // number of stages for the async global->shared
385
- // fetch pipeline
386
- const bool has_act_order, // whether act_order is enabled
387
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
388
- // a separate quantization scale
396
+template <typename scalar_t, // compute dtype, half or nv_float16
397
+ const int num_bits, // number of bits used for weights
398
+ const int threads, // number of threads in a threadblock
399
+ const int thread_m_blocks, // number of 16x16 blocks in the m
400
+ // dimension (batchsize) of the
401
+ // threadblock
402
+ const int thread_n_blocks, // same for n dimension (output)
403
+ const int thread_k_blocks, // same for k dimension (reduction)
404
+ const int stages, // number of stages for the async global->shared
405
+ // fetch pipeline
406
+ const bool has_act_order, // whether act_order is enabled
407
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
408
+ // with a separate quantization scale
409
>
390
-__global__ void
391
-Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
392
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
393
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
394
- const int4 *__restrict__ scales_ptr, // fp16 quantization scales of shape
395
- // (k/groupsize)xn
396
- const int *__restrict__ g_idx, // int32 group indices of shape k
397
- int num_groups, // number of scale groups per output channel
398
- int prob_m, // batch dimension m
399
- int prob_n, // output dimension n
400
- int prob_k, // reduction dimension k
401
- int *locks // extra global storage for barrier synchronization
410
+__global__ void Marlin(
411
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
412
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
413
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
414
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
415
+ // (k/groupsize)xn
416
+ const int* __restrict__ g_idx, // int32 group indices of shape k
417
+ int num_groups, // number of scale groups per output channel
418
+ int prob_m, // batch dimension m
419
+ int prob_n, // output dimension n
420
+ int prob_k, // reduction dimension k
421
+ int* locks // extra global storage for barrier synchronization
422
) {
423
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
424
// same size, which might involve multiple column "slices" (of width 16 *
@@ -445,11 +465,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
465
int slice_row = (iters * blockIdx.x) % k_tiles;
466
int slice_col_par = (iters * blockIdx.x) / k_tiles;
467
int slice_col = slice_col_par;
448
- int slice_iters; // number of threadblock tiles in the current slice
468
+ int slice_iters; // number of threadblock tiles in the current slice
469
int slice_count =
450
- 0; // total number of active threadblocks in the current slice
451
- int slice_idx; // index of threadblock in current slice; numbered bottom to
452
- // top
470
+ 0; // total number of active threadblocks in the current slice
471
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
472
+ // top
473
474
// We can easily implement parallel problem execution by just remapping
475
// indices and advancing global pointers
@@ -465,27 +485,22 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
485
auto init_slice = [&]() {
486
slice_iters =
487
iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
468
- if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
469
- slice_iters = 0;
470
- if (slice_iters == 0)
471
- return;
472
- if (slice_row + slice_iters > k_tiles)
473
- slice_iters = k_tiles - slice_row;
488
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
489
+ if (slice_iters == 0) return;
490
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
491
slice_count = 1;
492
slice_idx = 0;
493
int col_first = iters * div_ceil(k_tiles * slice_col_par, iters);
494
if (col_first <= k_tiles * (slice_col_par + 1)) {
495
int col_off = col_first - k_tiles * slice_col_par;
496
slice_count = div_ceil(k_tiles - col_off, iters);
480
- if (col_off > 0)
481
- slice_count++;
497
+ if (col_off > 0) slice_count++;
498
int delta_first = iters * blockIdx.x - col_first;
499
if (delta_first < 0 || (col_off == 0 && delta_first == 0))
500
slice_idx = slice_count - 1;
501
else {
502
slice_idx = slice_count - 1 - delta_first / iters;
487
- if (col_off > 0)
488
- slice_idx--;
503
+ if (col_off > 0) slice_idx--;
504
}
505
}
506
if (slice_col == n_tiles) {
@@ -605,7 +620,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
620
// needed if there are more threads than required for a certain tilesize or
621
// when the batchsize is not a multiple of 16.
622
bool a_sh_wr_pred[a_sh_wr_iters];
608
-#pragma unroll
623
+ #pragma unroll
624
for (int i = 0; i < a_sh_wr_iters; i++)
625
a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
626
@@ -623,13 +638,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
638
// loop unrolls, all shared memory accesses are static, we simply precompute
639
// both transformed reads and writes.
640
int a_sh_wr_trans[a_sh_wr_iters];
626
-#pragma unroll
641
+ #pragma unroll
642
for (int i = 0; i < a_sh_wr_iters; i++)
643
a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
644
int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
630
-#pragma unroll
645
+ #pragma unroll
646
for (int i = 0; i < b_sh_wr_iters; i++) {
632
-#pragma unroll
647
+ #pragma unroll
648
for (int j = 0; j < thread_m_blocks; j++)
649
a_sh_rd_trans[i][j] =
650
transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -639,30 +654,30 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
654
// runtime; we break dependencies between subsequent accesses with a tile by
655
// maintining multiple pointers (we have enough registers), a tiny
656
// optimization.
642
- const int4 *B_ptr[b_sh_wr_iters];
643
-#pragma unroll
657
+ const int4* B_ptr[b_sh_wr_iters];
658
+ #pragma unroll
659
for (int i = 0; i < b_sh_wr_iters; i++)
660
B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
661
662
extern __shared__ int4 sh[];
663
// Shared memory storage for global fetch pipelines.
649
- int4 *sh_a = sh;
650
- int4 *sh_b = sh_a + (stages * a_sh_stage);
651
- int4 *sh_g_idx = sh_b + (stages * b_sh_stage);
652
- int4 *sh_s = sh_g_idx + (stages * g_idx_stage);
664
+ int4* sh_a = sh;
665
+ int4* sh_b = sh_a + (stages * a_sh_stage);
666
+ int4* sh_g_idx = sh_b + (stages * b_sh_stage);
667
+ int4* sh_s = sh_g_idx + (stages * g_idx_stage);
668
669
// Register storage for double buffer of shared memory reads.
670
FragA frag_a[2][thread_m_blocks];
671
I4 frag_b_quant[2][b_thread_vecs];
672
FragC frag_c[thread_m_blocks][4][2];
658
- FragS frag_s[2][4]; // No act-order
659
- FragS act_frag_s[2][4][4]; // For act-order
673
+ FragS frag_s[2][4]; // No act-order
674
+ FragS act_frag_s[2][4][4]; // For act-order
675
676
// Zero accumulators.
677
auto zero_accums = [&]() {
663
-#pragma unroll
678
+ #pragma unroll
679
for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
665
- reinterpret_cast<float *>(frag_c)[i] = 0;
680
+ reinterpret_cast<float*>(frag_c)[i] = 0;
681
};
682
683
int sh_first_group_id = -1;
@@ -706,18 +721,18 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
721
// shared memory pipeline location.
722
auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
723
if (pred) {
709
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
710
-#pragma unroll
724
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
725
+ #pragma unroll
726
for (int i = 0; i < a_sh_wr_iters; i++) {
727
cp_async4_pred(
728
&sh_a_stage[a_sh_wr_trans[i]],
729
&A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
730
a_sh_wr_pred[i]);
731
}
717
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
718
-#pragma unroll
732
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
733
+ #pragma unroll
734
for (int i = 0; i < b_sh_wr_iters; i++) {
720
-#pragma unroll
735
+ #pragma unroll
736
for (int j = 0; j < b_thread_vecs; j++) {
737
cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
738
}
@@ -730,10 +745,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
745
int full_pipe = a_off;
746
int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
747
if (cur_k < prob_k && cur_k < slice_k_finish) {
733
- int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
748
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
749
735
- int4 const *cur_g_idx_stage_ptr =
736
- reinterpret_cast<int4 const *>(&g_idx[cur_k]);
750
+ int4 const* cur_g_idx_stage_ptr =
751
+ reinterpret_cast<int4 const*>(&g_idx[cur_k]);
752
753
if (threadIdx.x < g_idx_stage) {
754
cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
@@ -742,7 +757,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
757
}
758
} else {
759
if constexpr (group_blocks != -1) {
745
- int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
760
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
761
762
if constexpr (group_blocks >= thread_k_blocks) {
763
// Only fetch scales if this tile starts a new group
@@ -782,15 +797,16 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
797
// Load the next sub-tile from the current location in the shared memory pipe
798
// into the current register buffer.
799
auto fetch_to_registers = [&](int k, int pipe) {
785
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
786
-#pragma unroll
800
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
801
+ #pragma unroll
802
for (int i = 0; i < thread_m_blocks; i++)
788
- ldsm4<scalar_t>(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
789
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
803
+ ldsm4<scalar_t>(frag_a[k % 2][i],
804
+ &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
805
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
806
791
-#pragma unroll
807
+ #pragma unroll
808
for (int i = 0; i < b_thread_vecs; i++) {
793
- frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
809
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
810
&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
811
}
812
};
@@ -805,8 +821,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
821
return;
822
}
823
808
- int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
809
- int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
824
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
825
+ int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
826
827
int group_id_1 = sh_g_idx_int_ptr[0];
828
int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
@@ -822,10 +838,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
838
// No act-order case
839
if constexpr (group_blocks != -1) {
840
if constexpr (group_blocks >= thread_k_blocks) {
825
- int4 *sh_s_stage =
841
+ int4* sh_s_stage =
842
sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
843
(pipe / (group_blocks / thread_k_blocks)));
828
- reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
844
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
845
} else {
846
int warp_id = threadIdx.x / 32;
847
int n_warps = thread_n_blocks / 4;
@@ -838,9 +854,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
854
int k_blocks = cur_k / 16;
855
int cur_group_id = k_blocks / group_blocks;
856
841
- int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
857
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
858
843
- reinterpret_cast<int4 *>(&frag_s[k % 2])[0] =
859
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
860
sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
861
}
862
}
@@ -867,7 +883,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
883
// thread-id)
884
int warp_id = threadIdx.x / 32;
885
int n_warps =
870
- thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
886
+ thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
887
888
int warp_row = warp_id / n_warps;
889
int warp_col = warp_id % n_warps;
@@ -875,7 +891,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
891
cur_k += warp_row * 16;
892
893
int th_id = threadIdx.x % 32;
878
- cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
894
+ cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
895
896
int s_col_shift =
897
/*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
@@ -883,45 +899,44 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
899
900
if (is_same_group[pipe]) {
901
if (k % 2 == 0) {
886
- *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
902
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
903
sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
904
s_col_shift];
905
} else {
890
- *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0]))) =
891
- *(reinterpret_cast<int4 *>(&(act_frag_s[(k - 1) % 2][0][0])));
906
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
907
+ *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
908
}
909
910
for (int i = 1; i < 4; i++) {
895
- *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
896
- *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][0][0])));
911
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
912
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
913
}
914
return;
915
}
916
901
- int4 *sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
902
- int *sh_g_idx_int_ptr = reinterpret_cast<int *>(sh_g_idx_stage);
917
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
918
+ int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
919
920
constexpr int k_frag_offsets[4] = {0, 1, 8,
905
- 9}; // Tensor core offsets per thread
921
+ 9}; // Tensor core offsets per thread
922
907
-#pragma unroll
923
+ #pragma unroll
924
for (int i = 0; i < 4; i++) {
909
-
925
int actual_k = cur_k + k_frag_offsets[i];
926
927
int group_id = sh_g_idx_int_ptr[actual_k];
928
int rel_group_id = group_id - sh_first_group_id;
929
915
- *(reinterpret_cast<int4 *>(&(act_frag_s[k % 2][i][0]))) =
930
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
931
sh_s[rel_group_id * s_sh_stride + s_col_shift];
932
}
933
};
934
935
// Execute the actual tensor core matmul of a sub-tile.
936
auto matmul = [&](int k) {
922
-// We have the m dimension as the inner loop in order to encourage overlapping
923
-// dequantization and matmul operations.
924
-#pragma unroll
937
+ // We have the m dimension as the inner loop in order to encourage overlapping
938
+ // dequantization and matmul operations.
939
+ #pragma unroll
940
for (int j = 0; j < 4; j++) {
941
FragB frag_b0;
942
FragB frag_b1;
@@ -933,7 +948,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
948
frag_b1 = dequant_4bit<scalar_t>(b_quant_shift);
949
950
} else {
936
- int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
951
+ int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
952
int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
953
int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
954
@@ -943,8 +958,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
958
959
// Apply scale to frag_b0
960
if constexpr (has_act_order) {
946
- scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
947
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
961
+ scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j],
962
+ act_frag_s[k % 2][1][j], act_frag_s[k % 2][2][j],
963
+ act_frag_s[k % 2][3][j], 0);
964
} else {
965
if constexpr (group_blocks != -1) {
966
scale<scalar_t>(frag_b0, frag_s[k % 2][j], 0);
@@ -953,8 +969,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
969
970
// Apply scale to frag_b1
971
if constexpr (has_act_order) {
956
- scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
957
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
972
+ scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
973
+ act_frag_s[k % 2][1][j], act_frag_s[k % 2][2][j],
974
+ act_frag_s[k % 2][3][j], 1);
975
976
} else {
977
if constexpr (group_blocks != -1) {
@@ -962,7 +979,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
979
}
980
}
981
965
-#pragma unroll
982
+ #pragma unroll
983
for (int i = 0; i < thread_m_blocks; i++) {
984
mma<scalar_t>(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
985
mma<scalar_t>(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
@@ -987,38 +1004,38 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1004
// unnecessary read or write iterations, e.g., for two warps we write only
1005
// once by warp 1 and read only once by warp 0.
1006
990
-#pragma unroll
1007
+ #pragma unroll
1008
for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
992
-#pragma unroll
1009
+ #pragma unroll
1010
for (int i = red_off; i > 0; i /= 2) {
1011
if (i <= red_idx && red_idx < 2 * i) {
995
-#pragma unroll
1012
+ #pragma unroll
1013
for (int j = 0; j < 4 * 2; j++) {
1014
int red_sh_wr =
1015
red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
1016
if (i < red_off) {
1000
- float *c_rd = reinterpret_cast<float *>(
1001
- &sh[red_sh_delta * j + red_sh_rd]);
1002
- float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
1003
-#pragma unroll
1017
+ float* c_rd =
1018
+ reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
1019
+ float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
1020
+ #pragma unroll
1021
for (int k = 0; k < 4; k++)
1005
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
1022
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
1023
c_rd[k] + c_wr[k];
1024
}
1025
sh[red_sh_wr] =
1009
- reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
1026
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
1027
}
1028
}
1029
__syncthreads();
1030
}
1031
if (red_idx == 0) {
1015
-#pragma unroll
1032
+ #pragma unroll
1033
for (int i = 0; i < 4 * 2; i++) {
1017
- float *c_rd =
1018
- reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
1019
-#pragma unroll
1034
+ float* c_rd =
1035
+ reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
1036
+ #pragma unroll
1037
for (int j = 0; j < 4; j++)
1021
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
1038
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
1039
c_rd[j];
1040
}
1041
}
@@ -1049,39 +1066,39 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1066
int row = (threadIdx.x % 32) / 4;
1067
1068
if (!first) {
1052
-// Interestingly, doing direct global accesses here really seems to mess up the
1053
-// compiler and lead to slowdowns, hence we also use async-copies even though
1054
-// these fetches are not actually asynchronous.
1055
-#pragma unroll
1069
+ // Interestingly, doing direct global accesses here really seems to mess up
1070
+ // the compiler and lead to slowdowns, hence we also use async-copies even
1071
+ // though these fetches are not actually asynchronous.
1072
+ #pragma unroll
1073
for (int i = 0; i < thread_m_blocks * 4; i++) {
1057
- cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
1058
- &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
1059
- c_gl_wr_delta_i * (i % 2)],
1060
- i < (thread_m_blocks - 1) * 4 ||
1061
- 8 * (i / 2) + row < prob_m);
1074
+ cp_async4_pred(
1075
+ &sh[c_sh_wr + c_sh_wr_delta * i],
1076
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
1077
+ c_gl_wr_delta_i * (i % 2)],
1078
+ i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
1079
}
1080
cp_async_fence();
1081
cp_async_wait<0>();
1082
}
1083
1067
-#pragma unroll
1084
+ #pragma unroll
1085
for (int i = 0; i < thread_m_blocks * 4; i++) {
1086
if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
1087
if (!first) {
1088
int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
1072
-#pragma unroll
1089
+ #pragma unroll
1090
for (int j = 0; j < 2 * 4; j++) {
1074
- reinterpret_cast<float *>(
1091
+ reinterpret_cast<float*>(
1092
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
1076
- Dtype::num2float(reinterpret_cast<scalar_t *>(&c_red)[j]);
1093
+ Dtype::num2float(reinterpret_cast<scalar_t*>(&c_red)[j]);
1094
}
1095
}
1096
if (!last) {
1097
int4 c;
1081
-#pragma unroll
1098
+ #pragma unroll
1099
for (int j = 0; j < 2 * 4; j++) {
1083
- reinterpret_cast<scalar_t *>(&c)[j] =
1084
- Dtype::float2num(reinterpret_cast<float *>(
1100
+ reinterpret_cast<scalar_t*>(&c)[j] =
1101
+ Dtype::float2num(reinterpret_cast<float*>(
1102
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1103
}
1104
C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -1115,8 +1132,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1132
1133
// We first reorder in shared memory to guarantee the most efficient final
1134
// global write patterns
1118
- auto write = [&](int idx, float c0, float c1, FragS &s) {
1119
- scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1135
+ auto write = [&](int idx, float c0, float c1, FragS& s) {
1136
+ scalar_t2 res =
1137
+ Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
1138
1139
// For per-column quantization we finally apply the scale here (only for
1140
// 4-bit)
@@ -1124,13 +1142,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1142
res = __hmul2(res, s[0]);
1143
}
1144
1127
- ((scalar_t2 *)sh)[idx] = res;
1145
+ ((scalar_t2*)sh)[idx] = res;
1146
};
1147
1148
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1131
-#pragma unroll
1149
+ #pragma unroll
1150
for (int i = 0; i < thread_m_blocks; i++) {
1133
-#pragma unroll
1151
+ #pragma unroll
1152
for (int j = 0; j < 4; j++) {
1153
int wr = c_sh_wr + 8 * j;
1154
write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
@@ -1147,7 +1165,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1165
}
1166
__syncthreads();
1167
1150
-#pragma unroll
1168
+ #pragma unroll
1169
for (int i = 0;
1170
i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1171
i++) {
@@ -1162,7 +1180,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1180
// Start global fetch and register load pipelines.
1181
auto start_pipes = [&]() {
1182
1165
-#pragma unroll
1183
+ #pragma unroll
1184
for (int i = 0; i < stages - 1; i++) {
1185
if (has_act_order && i == 0) {
1186
int last_g_idx = slice_k_start + stages * tb_k * 2;
@@ -1193,9 +1211,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1211
// have even length meaning that the next iteration will always start at
1212
// index 0.
1213
1196
-#pragma unroll
1214
+ #pragma unroll
1215
for (int pipe = 0; pipe < stages;) {
1198
-#pragma unroll
1216
+ #pragma unroll
1217
for (int k = 0; k < b_sh_wr_iters; k++) {
1218
fetch_to_registers(k + 1, pipe % stages);
1219
fetch_scales_to_registers(k + 1, pipe);
@@ -1261,8 +1279,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1279
cp_async_wait<0>();
1280
__syncthreads();
1281
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264
- reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265
- reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1282
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1283
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1284
}
1285
1286
} else {
@@ -1270,8 +1288,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1288
cp_async_wait<0>();
1289
__syncthreads();
1290
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273
- reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274
- reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1291
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1292
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1293
}
1294
}
1295
}
@@ -1282,31 +1300,35 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1300
// overflow in fp16)
1301
if constexpr (!has_act_order && group_blocks == -1 && num_bits == 8) {
1302
if (threadIdx.x / 32 < thread_n_blocks / 4) {
1285
-#pragma unroll
1303
+ #pragma unroll
1304
for (int i = 0; i < thread_m_blocks; i++) {
1287
-#pragma unroll
1305
+ #pragma unroll
1306
for (int j = 0; j < 4; j++) {
1289
- scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][0]),
1290
- frag_s[j / 2][2 * (j % 2) + 0]);
1291
- scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][0][2]),
1292
- frag_s[j / 2][2 * (j % 2) + 0]);
1293
-
1294
- scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][0]),
1295
- frag_s[j / 2][2 * (j % 2) + 1]);
1296
- scale_float<scalar_t>(reinterpret_cast<float *>(&frag_c[i][j][1][2]),
1297
- frag_s[j / 2][2 * (j % 2) + 1]);
1307
+ scale_float<scalar_t>(
1308
+ reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1309
+ frag_s[j / 2][2 * (j % 2) + 0]);
1310
+ scale_float<scalar_t>(
1311
+ reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1312
+ frag_s[j / 2][2 * (j % 2) + 0]);
1313
+
1314
+ scale_float<scalar_t>(
1315
+ reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1316
+ frag_s[j / 2][2 * (j % 2) + 1]);
1317
+ scale_float<scalar_t>(
1318
+ reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1319
+ frag_s[j / 2][2 * (j % 2) + 1]);
1320
}
1321
}
1322
}
1323
}
1324
1303
- if (slice_count > 1) { // only globally reduce if there is more than one
1304
- // block in a slice
1325
+ if (slice_count > 1) { // only globally reduce if there is more than one
1326
+ // block in a slice
1327
barrier_acquire(&locks[slice_col], slice_idx);
1328
global_reduce(slice_idx == 0, last);
1329
barrier_release(&locks[slice_col], last);
1330
}
1309
- if (last) // only the last block in a slice actually writes the result
1331
+ if (last) // only the last block in a slice actually writes the result
1332
write_result();
1333
slice_row = 0;
1334
slice_col_par++;
@@ -1315,13 +1337,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1337
if (slice_iters) {
1338
a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1339
(threadIdx.x % a_gl_rd_delta_o);
1318
-#pragma unroll
1340
+ #pragma unroll
1341
for (int i = 0; i < b_sh_wr_iters; i++)
1342
B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1343
if (slice_col == 0) {
1322
-#pragma unroll
1323
- for (int i = 0; i < b_sh_wr_iters; i++)
1324
- B_ptr[i] -= b_gl_stride;
1344
+ #pragma unroll
1345
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1346
}
1347
1348
// Update slice k/n for scales loading
@@ -1341,23 +1362,24 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
1362
}
1363
}
1364
1344
-#define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1345
- HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1346
- else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1347
- thread_n_blocks == THREAD_N_BLOCKS && \
1348
- thread_k_blocks == THREAD_K_BLOCKS && \
1349
- has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1350
- num_threads == NUM_THREADS) { \
1351
- cudaFuncSetAttribute( \
1352
- Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1353
- THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1354
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1355
- Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1356
- THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, GROUP_BLOCKS> \
1357
- <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1358
- A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1359
- prob_k, locks); \
1360
- }
1365
+ #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1366
+ THREAD_K_BLOCKS, HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1367
+ else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
1368
+ thread_n_blocks == THREAD_N_BLOCKS && \
1369
+ thread_k_blocks == THREAD_K_BLOCKS && \
1370
+ has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1371
+ num_threads == NUM_THREADS) { \
1372
+ cudaFuncSetAttribute( \
1373
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1374
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1375
+ GROUP_BLOCKS>, \
1376
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1377
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
1378
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1379
+ GROUP_BLOCKS><<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1380
+ A_ptr, B_ptr, C_ptr, s_ptr, g_idx_ptr, num_groups, prob_m, prob_n, \
1381
+ prob_k, locks); \
1382
+ }
1383
1384
typedef struct {
1385
int thread_k;
@@ -1389,7 +1411,7 @@ thread_config_t large_batch_thread_configs[] = {
1411
1412
};
1413
1392
-int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1414
+int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
1415
int prob_n, int prob_k, int num_bits, int group_size,
1416
bool has_act_order, bool is_k_full) {
1417
bool cache_scales_chunk = has_act_order && !is_k_full;
@@ -1402,15 +1424,15 @@ int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1424
if (group_size == -1) {
1425
tb_groups = 1;
1426
} else if (group_size == 0) {
1405
- tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1427
+ tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
1428
} else {
1429
tb_groups = div_ceil(tb_k, group_size);
1430
}
1431
1432
if (cache_scales_chunk) {
1433
int load_groups =
1412
- tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1413
- load_groups = max(load_groups, 32); // We load at least 32 scale groups
1434
+ tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K
1435
+ load_groups = max(load_groups, 32); // We load at least 32 scale groups
1436
return load_groups * tb_n * 2;
1437
1438
} else {
@@ -1420,7 +1442,7 @@ int get_scales_cache_size(thread_config_t const &th_config, int prob_m,
1442
}
1443
}
1444
1423
-bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1445
+bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
1446
int prob_m, int prob_n, int prob_k, int num_bits,
1447
int scales_cache_size, int max_shared_mem) {
1448
int pack_factor = 32 / num_bits;
@@ -1451,12 +1473,12 @@ bool is_valid_cache_size(thread_config_t const &th_config, int max_m_blocks,
1473
1474
float pipe_size = (a_size + b_size) * pipe_stages;
1475
1454
- TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1476
+ TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1477
1478
return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1479
}
1480
1459
-bool is_valid_config(thread_config_t const &th_config, int max_m_blocks,
1481
+bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1482
int prob_m, int prob_n, int prob_k, int num_bits,
1483
int group_size, bool has_act_order, bool is_k_full,
1484
int max_shared_mem) {
@@ -1519,43 +1541,43 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1541
}
1542
}
1543
1522
- max_m_blocks--; // Process less M blocks per invocation to reduce cache
1523
- // usage
1544
+ max_m_blocks--; // Process less M blocks per invocation to reduce cache
1545
+ // usage
1546
}
1547
1548
return exec_config_t{0, {-1, -1, -1}};
1549
}
1550
1529
-#define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1530
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1531
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1532
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1533
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1534
- \
1535
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1536
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1537
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1538
- __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1539
- \
1540
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1541
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1542
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1543
- __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1544
- \
1545
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1546
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1547
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1548
- __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1549
- \
1550
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1551
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1552
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1553
- __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1551
+ #define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1552
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1553
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1554
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1555
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1556
+ \
1557
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1558
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1559
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1560
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1561
+ \
1562
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1563
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1564
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1565
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1566
+ \
1567
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1568
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1569
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1570
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1571
+ \
1572
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1573
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1574
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1575
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1576
1577
template <typename scalar_t>
1556
-void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1557
- void *g_idx, void *perm, void *a_tmp, int prob_m,
1558
- int prob_n, int prob_k, void *workspace, int num_bits,
1578
+void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s,
1579
+ void* g_idx, void* perm, void* a_tmp, int prob_m,
1580
+ int prob_n, int prob_k, void* workspace, int num_bits,
1581
bool has_act_order, bool is_k_full, int num_groups,
1582
int group_size, int dev, cudaStream_t stream, int thread_k,
1583
int thread_n, int sms, int max_par) {
@@ -1639,15 +1661,15 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1661
}
1662
}
1663
1642
- const int4 *A_ptr = (const int4 *)A;
1643
- const int4 *B_ptr = (const int4 *)B;
1644
- int4 *C_ptr = (int4 *)C;
1645
- const int4 *s_ptr = (const int4 *)s;
1646
- const int *g_idx_ptr = (const int *)g_idx;
1647
- const int *perm_ptr = (const int *)perm;
1648
- int4 *a_tmp_ptr = (int4 *)a_tmp;
1664
+ const int4* A_ptr = (const int4*)A;
1665
+ const int4* B_ptr = (const int4*)B;
1666
+ int4* C_ptr = (int4*)C;
1667
+ const int4* s_ptr = (const int4*)s;
1668
+ const int* g_idx_ptr = (const int*)g_idx;
1669
+ const int* perm_ptr = (const int*)perm;
1670
+ int4* a_tmp_ptr = (int4*)a_tmp;
1671
1650
- int *locks = (int *)workspace;
1672
+ int* locks = (int*)workspace;
1673
1674
if (has_act_order) {
1675
// Permute A columns
@@ -1673,8 +1695,7 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1695
// Note that parallel > 1 currently only works for inputs without any
1696
// padding
1697
par = (16 * thread_m_blocks - pad) / (16 * exec_cfg.max_m_blocks);
1676
- if (par > max_par)
1677
- par = max_par;
1698
+ if (par > max_par) par = max_par;
1699
prob_m = (16 * exec_cfg.max_m_blocks) * par;
1700
i += exec_cfg.max_m_blocks * (par - 1);
1701
thread_m_blocks = exec_cfg.max_m_blocks;
@@ -1709,11 +1730,11 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1730
}
1731
}
1732
1712
-} // namespace gptq_marlin
1733
+} // namespace gptq_marlin
1734
1714
-torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1715
- torch::Tensor &b_scales, torch::Tensor &g_idx,
1716
- torch::Tensor &perm, torch::Tensor &workspace,
1735
+torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1736
+ torch::Tensor& b_scales, torch::Tensor& g_idx,
1737
+ torch::Tensor& perm, torch::Tensor& workspace,
1738
int64_t num_bits, int64_t size_m, int64_t size_n,
1739
int64_t size_k, bool is_k_full) {
1740
// Verify num_bits
@@ -1824,18 +1845,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1845
int dev = a.get_device();
1846
if (a.scalar_type() == at::ScalarType::Half) {
1847
gptq_marlin::marlin_mm_f16i4<half>(
1827
- a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(), b_scales.data_ptr<at::Half>(),
1828
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n,
1829
- size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1830
- num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1831
- thread_k, thread_n, sms, gptq_marlin::max_par);
1848
+ a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
1849
+ b_scales.data_ptr<at::Half>(), g_idx.data_ptr(), perm.data_ptr(),
1850
+ a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
1851
+ workspace.data_ptr(), num_bits, has_act_order, is_k_full, num_groups,
1852
+ group_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1853
+ thread_n, sms, gptq_marlin::max_par);
1854
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
1855
gptq_marlin::marlin_mm_f16i4<nv_bfloat16>(
1834
- a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(), c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1835
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n,
1836
- size_k, workspace.data_ptr(), num_bits, has_act_order, is_k_full,
1837
- num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
1838
- thread_k, thread_n, sms, gptq_marlin::max_par);
1856
+ a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
1857
+ c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
1858
+ g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(),
1859
+ size_m, size_n, size_k, workspace.data_ptr(), num_bits, has_act_order,
1860
+ is_k_full, num_groups, group_size, dev,
1861
+ at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1862
+ gptq_marlin::max_par);
1863
} else {
1864
TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
1865
}
csrc/quantization/gptq_marlin/gptq_marlin.cuh
+28
-22
@@ -11,22 +11,23 @@
11
12
namespace gptq_marlin {
13
14
-// 8 warps are a good choice since every SM has 4 schedulers and having more than 1 warp per
15
-// schedule allows some more latency hiding. At the same time, we want relatively few warps to have
16
-// many registers per warp and small tiles.
14
+// 8 warps are a good choice since every SM has 4 schedulers and having more
15
+// than 1 warp per schedule allows some more latency hiding. At the same time,
16
+// we want relatively few warps to have many registers per warp and small tiles.
17
static constexpr int default_threads = 256;
18
19
-static constexpr int pipe_stages = 4; // 4 pipeline stages fit into shared memory
19
+static constexpr int pipe_stages =
20
+ 4; // 4 pipeline stages fit into shared memory
21
22
static constexpr int min_thread_n = 64;
23
static constexpr int min_thread_k = 64;
24
25
static constexpr int tile_size = 16;
25
-static constexpr int max_par = 16;
26
+static constexpr int max_par = 16;
27
28
template <typename T, int n>
29
struct Vec {
29
- T elems[n];
30
+ T elems[n];
31
__device__ T& operator[](int i) { return elems[i]; }
32
};
33
@@ -35,30 +36,35 @@ using I4 = Vec<int, 4>;
36
constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
37
38
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
38
- // No support for async
39
+// No support for async
40
#else
41
41
-__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool pred = true) {
42
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
43
+ bool pred = true) {
44
const int BYTES = 16;
43
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
44
- asm volatile("{\n"
45
- " .reg .pred p;\n"
46
- " setp.ne.b32 p, %0, 0;\n"
47
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
48
- "}\n" ::"r"((int)pred),
49
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
45
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
46
+ asm volatile(
47
+ "{\n"
48
+ " .reg .pred p;\n"
49
+ " setp.ne.b32 p, %0, 0;\n"
50
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
51
+ "}\n" ::"r"((int)pred),
52
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
53
}
54
55
__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
56
const int BYTES = 16;
54
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55
- asm volatile("{\n"
56
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
57
- "}\n" ::"r"(smem),
58
- "l"(glob_ptr), "n"(BYTES));
57
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
58
+ asm volatile(
59
+ "{\n"
60
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
61
+ "}\n" ::"r"(smem),
62
+ "l"(glob_ptr), "n"(BYTES));
63
}
64
61
-__device__ inline void cp_async_fence() { asm volatile("cp.async.commit_group;\n" ::); }
65
+__device__ inline void cp_async_fence() {
66
+ asm volatile("cp.async.commit_group;\n" ::);
67
+}
68
69
template <int n>
70
__device__ inline void cp_async_wait() {
@@ -67,4 +73,4 @@ __device__ inline void cp_async_wait() {
73
74
#endif
75
70
-} // namespace gptq_marlin
76
+} // namespace gptq_marlin
csrc/quantization/gptq_marlin/gptq_marlin_dtypes.cuh
+52
-37
@@ -5,58 +5,73 @@
5
#include <cuda_fp16.h>
6
#include <cuda_bf16.h>
7
8
-
8
namespace gptq_marlin {
9
10
template <typename scalar_t>
12
-class ScalarType {
13
-};
11
+class ScalarType {};
12
13
template <>
14
class ScalarType<half> {
17
-public:
18
- using scalar_t = half;
19
- using scalar_t2 = half2;
20
-
21
- // Matrix fragments for tensor core instructions; their precise layout is
22
- // documented here:
23
- // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
24
- using FragA = Vec<half2, 4>;
25
- using FragB = Vec<half2, 2>;
26
- using FragC = Vec<float, 4>;
27
- using FragS = Vec<half2, 1>;
28
-
29
- static __device__ float inline num2float(const half x) { return __half2float(x); }
30
-
31
- static __device__ half2 inline num2num2(const half x) { return __half2half2(x); }
32
-
33
- static __device__ half2 inline nums2num2(const half x1, const half x2) { return __halves2half2(x1, x2); }
34
-
35
- static __host__ __device__ half inline float2num(const float x) { return __float2half(x); }
15
+ public:
16
+ using scalar_t = half;
17
+ using scalar_t2 = half2;
18
+
19
+ // Matrix fragments for tensor core instructions; their precise layout is
20
+ // documented here:
21
+ // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
22
+ using FragA = Vec<half2, 4>;
23
+ using FragB = Vec<half2, 2>;
24
+ using FragC = Vec<float, 4>;
25
+ using FragS = Vec<half2, 1>;
26
+
27
+ static __device__ float inline num2float(const half x) {
28
+ return __half2float(x);
29
+ }
30
+
31
+ static __device__ half2 inline num2num2(const half x) {
32
+ return __half2half2(x);
33
+ }
34
+
35
+ static __device__ half2 inline nums2num2(const half x1, const half x2) {
36
+ return __halves2half2(x1, x2);
37
+ }
38
+
39
+ static __host__ __device__ half inline float2num(const float x) {
40
+ return __float2half(x);
41
+ }
42
};
43
44
template <>
45
class ScalarType<nv_bfloat16> {
40
-public:
41
- using scalar_t = nv_bfloat16;
42
- using scalar_t2 = nv_bfloat162;
46
+ public:
47
+ using scalar_t = nv_bfloat16;
48
+ using scalar_t2 = nv_bfloat162;
49
44
- using FragA = Vec<nv_bfloat162, 4>;
45
- using FragB = Vec<nv_bfloat162, 2>;
46
- using FragC = Vec<float, 4>;
47
- using FragS = Vec<nv_bfloat162, 1>;
50
+ using FragA = Vec<nv_bfloat162, 4>;
51
+ using FragB = Vec<nv_bfloat162, 2>;
52
+ using FragC = Vec<float, 4>;
53
+ using FragS = Vec<nv_bfloat162, 1>;
54
55
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
50
- static __device__ float inline num2float(const nv_bfloat16 x) { return __bfloat162float(x); }
51
-
52
- static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { return __bfloat162bfloat162(x); }
53
-
54
- static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { return __halves2bfloat162(x1, x2); }
55
-
56
- static __host__ __device__ nv_bfloat16 inline float2num(const float x) { return __float2bfloat16(x); }
56
+ static __device__ float inline num2float(const nv_bfloat16 x) {
57
+ return __bfloat162float(x);
58
+ }
59
+
60
+ static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) {
61
+ return __bfloat162bfloat162(x);
62
+ }
63
+
64
+ static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1,
65
+ const nv_bfloat16 x2) {
66
+ return __halves2bfloat162(x1, x2);
67
+ }
68
+
69
+ static __host__ __device__ nv_bfloat16 inline float2num(const float x) {
70
+ return __float2bfloat16(x);
71
+ }
72
#endif
73
};
74
60
-}
75
+} // namespace gptq_marlin
76
77
#endif
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+46
-48
@@ -12,14 +12,14 @@ static constexpr int tile_n_size = tile_k_size * 4;
12
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
13
14
template <int const num_threads, int const num_bits, bool const has_perm>
15
-__global__ void
16
-marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
17
- uint32_t const *__restrict__ perm_ptr,
18
- uint32_t *__restrict__ out_ptr, int size_k, int size_n) {}
15
+__global__ void marlin_repack_kernel(
16
+ uint32_t const* __restrict__ b_q_weight_ptr,
17
+ uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
18
+ int size_k, int size_n) {}
19
20
-} // namespace gptq_marlin
20
+} // namespace gptq_marlin
21
22
-torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
22
+torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
23
int64_t size_k, int64_t size_n,
24
int64_t num_bits) {
25
TORCH_CHECK_NOT_IMPLEMENTED(
@@ -30,10 +30,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
30
#else
31
32
template <int const num_threads, int const num_bits, bool const has_perm>
33
-__global__ void
34
-marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
35
- uint32_t const *__restrict__ perm_ptr,
36
- uint32_t *__restrict__ out_ptr, int size_k, int size_n) {
33
+__global__ void marlin_repack_kernel(
34
+ uint32_t const* __restrict__ b_q_weight_ptr,
35
+ uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
36
+ int size_k, int size_n) {
37
constexpr int pack_factor = 32 / num_bits;
38
39
int k_tiles = size_k / tile_k_size;
@@ -61,8 +61,8 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
61
62
constexpr int perm_size = tile_k_size / 4;
63
64
- int4 *sh_perm_ptr = sh;
65
- int4 *sh_pipe_ptr = sh_perm_ptr;
64
+ int4* sh_perm_ptr = sh;
65
+ int4* sh_pipe_ptr = sh_perm_ptr;
66
if constexpr (has_perm) {
67
sh_pipe_ptr += perm_size;
68
}
@@ -76,7 +76,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
76
auto load_perm_to_shared = [&](int k_tile_id) {
77
int first_k_int4 = (k_tile_id * tile_k_size) / 4;
78
79
- int4 const *perm_int4_ptr = reinterpret_cast<int4 const *>(perm_ptr);
79
+ int4 const* perm_int4_ptr = reinterpret_cast<int4 const*>(perm_ptr);
80
81
if (threadIdx.x < perm_size) {
82
sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x];
@@ -92,22 +92,22 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
92
93
int first_n = n_tile_id * tile_n_size;
94
95
- int4 *sh_ptr = sh_pipe_ptr + stage_size * pipe;
95
+ int4* sh_ptr = sh_pipe_ptr + stage_size * pipe;
96
97
if constexpr (has_perm) {
98
if (threadIdx.x < stage_size) {
99
int k_id = threadIdx.x / stage_n_threads;
100
int n_id = threadIdx.x % stage_n_threads;
101
102
- uint32_t const *sh_perm_int_ptr =
103
- reinterpret_cast<uint32_t const *>(sh_perm_ptr);
102
+ uint32_t const* sh_perm_int_ptr =
103
+ reinterpret_cast<uint32_t const*>(sh_perm_ptr);
104
105
int src_k = sh_perm_int_ptr[k_id];
106
int src_k_packed = src_k / pack_factor;
107
108
cp_async4(
109
&sh_ptr[k_id * stage_n_threads + n_id],
110
- reinterpret_cast<int4 const *>(&(
110
+ reinterpret_cast<int4 const*>(&(
111
b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
112
}
113
@@ -120,7 +120,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
120
int first_k_packed = first_k / pack_factor;
121
122
cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
123
- reinterpret_cast<int4 const *>(
123
+ reinterpret_cast<int4 const*>(
124
&(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
125
first_n + (n_id * 4)])));
126
}
@@ -151,10 +151,10 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
151
constexpr int sh_stride = 64;
152
constexpr uint32_t mask = (1 << num_bits) - 1;
153
154
- int4 *sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155
- uint32_t *sh_stage_int_ptr = reinterpret_cast<uint32_t *>(sh_stage_ptr);
154
+ int4* sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
155
+ uint32_t* sh_stage_int_ptr = reinterpret_cast<uint32_t*>(sh_stage_ptr);
156
157
- uint32_t *sh_perm_int_ptr = reinterpret_cast<uint32_t *>(sh_perm_ptr);
157
+ uint32_t* sh_perm_int_ptr = reinterpret_cast<uint32_t*>(sh_perm_ptr);
158
159
uint32_t vals[8];
160
@@ -176,17 +176,16 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
176
}
177
178
} else {
179
-
179
uint32_t b1_vals[tile_ints];
180
uint32_t b2_vals[tile_ints];
181
183
-#pragma unroll
182
+ #pragma unroll
183
for (int i = 0; i < tile_ints; i++) {
184
b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
185
b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
186
}
187
189
-#pragma unroll
188
+ #pragma unroll
189
for (int i = 0; i < 4; i++) {
190
int cur_elem = tc_row + tc_offsets[i];
191
int cur_int = cur_elem / pack_factor;
@@ -206,7 +205,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
205
constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
206
207
uint32_t res = 0;
209
-#pragma unroll
208
+ #pragma unroll
209
for (int i = 0; i < 8; i++) {
210
res |= vals[pack_idx[i]] << (i * 4);
211
}
@@ -218,7 +217,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
217
218
uint32_t res1 = 0;
219
uint32_t res2 = 0;
221
-#pragma unroll
220
+ #pragma unroll
221
for (int i = 0; i < 4; i++) {
222
res1 |= vals[pack_idx[i]] << (i * 8);
223
res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -230,14 +229,14 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
229
};
230
231
auto start_pipes = [&](int k_tile_id, int n_tile_id) {
233
-#pragma unroll
232
+ #pragma unroll
233
for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
234
fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
235
}
236
237
wait_for_stage();
238
};
240
-#pragma unroll
239
+ #pragma unroll
240
for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
241
int n_tile_id = 0;
242
@@ -248,7 +247,7 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
247
start_pipes(k_tile_id, n_tile_id);
248
249
while (n_tile_id < n_tiles) {
251
-#pragma unroll
250
+ #pragma unroll
251
for (int pipe = 0; pipe < repack_stages; pipe++) {
252
fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
253
n_tile_id + pipe + repack_stages - 1);
@@ -260,21 +259,21 @@ marlin_repack_kernel(uint32_t const *__restrict__ b_q_weight_ptr,
259
}
260
}
261
263
-} // namespace gptq_marlin
264
-
265
-#define CALL_IF(NUM_BITS, HAS_PERM) \
266
- else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
267
- cudaFuncSetAttribute( \
268
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
269
- NUM_BITS, HAS_PERM>, \
270
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
271
- gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
272
- HAS_PERM> \
273
- <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
274
- b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
275
- }
262
+} // namespace gptq_marlin
263
+
264
+ #define CALL_IF(NUM_BITS, HAS_PERM) \
265
+ else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
266
+ cudaFuncSetAttribute( \
267
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, \
268
+ NUM_BITS, HAS_PERM>, \
269
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
270
+ gptq_marlin::marlin_repack_kernel<gptq_marlin::repack_threads, NUM_BITS, \
271
+ HAS_PERM> \
272
+ <<<blocks, gptq_marlin::repack_threads, max_shared_mem, stream>>>( \
273
+ b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
274
+ }
275
277
-torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
276
+torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
277
int64_t size_k, int64_t size_n,
278
int64_t num_bits) {
279
// Verify compatibility with marlin tile of 16x64
@@ -318,11 +317,10 @@ torch::Tensor gptq_marlin_repack(torch::Tensor &b_q_weight, torch::Tensor &perm,
317
bool has_perm = perm.size(0) != 0;
318
319
// Get ptrs
321
- uint32_t const *b_q_weight_ptr =
322
- reinterpret_cast<uint32_t const *>(b_q_weight.data_ptr());
323
- uint32_t const *perm_ptr =
324
- reinterpret_cast<uint32_t const *>(perm.data_ptr());
325
- uint32_t *out_ptr = reinterpret_cast<uint32_t *>(out.data_ptr());
320
+ uint32_t const* b_q_weight_ptr =
321
+ reinterpret_cast<uint32_t const*>(b_q_weight.data_ptr());
322
+ uint32_t const* perm_ptr = reinterpret_cast<uint32_t const*>(perm.data_ptr());
323
+ uint32_t* out_ptr = reinterpret_cast<uint32_t*>(out.data_ptr());
324
325
// Get dev info
326
int dev = b_q_weight.get_device();
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+229
-231
@@ -25,7 +25,10 @@
25
26
#include <iostream>
27
28
-template <typename T> inline std::string str(T x) { return std::to_string(x); }
28
+template <typename T>
29
+inline std::string str(T x) {
30
+ return std::to_string(x);
31
+}
32
33
namespace marlin {
34
@@ -38,9 +41,10 @@ constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
41
// corresponding index accesses must be compile-time constants, which is why we
42
// extensively use `#pragma unroll` throughout the kernel code to guarantee
43
// this.
41
-template <typename T, int n> struct Vec {
44
+template <typename T, int n>
45
+struct Vec {
46
T elems[n];
43
- __device__ T &operator[](int i) { return elems[i]; }
47
+ __device__ T& operator[](int i) { return elems[i]; }
48
};
49
50
using I4 = Vec<int, 4>;
@@ -51,29 +55,32 @@ using I4 = Vec<int, 4>;
55
using FragA = Vec<half2, 4>;
56
using FragB = Vec<half2, 2>;
57
using FragC = Vec<float, 4>;
54
-using FragS = Vec<half2, 1>; // quantization scales
58
+using FragS = Vec<half2, 1>; // quantization scales
59
60
// Predicated asynchronous global->shared copy; used for inputs A where we apply
61
// predication to handle batchsizes that are not multiples of 16.
58
-__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
62
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63
bool pred = true) {
64
const int BYTES = 16;
65
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
62
- asm volatile("{\n"
63
- " .reg .pred p;\n"
64
- " setp.ne.b32 p, %0, 0;\n"
65
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
66
- "}\n" ::"r"((int)pred),
67
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
66
+ asm volatile(
67
+ "{\n"
68
+ " .reg .pred p;\n"
69
+ " setp.ne.b32 p, %0, 0;\n"
70
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71
+ "}\n" ::"r"((int)pred),
72
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
73
}
74
75
// Asynchronous global->shared copy
71
-__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
76
+__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77
const int BYTES = 16;
78
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
74
- asm volatile("{\n"
75
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
76
- "}\n" :: "r"(smem), "l"(glob_ptr), "n"(BYTES));
79
+ asm volatile(
80
+ "{\n"
81
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
82
+ "}\n" ::"r"(smem),
83
+ "l"(glob_ptr), "n"(BYTES));
84
}
85
86
// Async copy fence.
@@ -82,28 +89,30 @@ __device__ inline void cp_async_fence() {
89
}
90
91
// Wait until at most `n` async copy stages are still pending.
85
-template <int n> __device__ inline void cp_async_wait() {
92
+template <int n>
93
+__device__ inline void cp_async_wait() {
94
asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95
}
96
97
// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
98
// output/accumulation.
91
-__device__ inline void mma(const FragA &a_frag, const FragB &frag_b,
92
- FragC &frag_c) {
93
- const uint32_t *a = reinterpret_cast<const uint32_t *>(&a_frag);
94
- const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
95
- float *c = reinterpret_cast<float *>(&frag_c);
96
- asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
97
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
98
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
99
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]),
100
- "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
99
+__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
100
+ FragC& frag_c) {
101
+ const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
102
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
103
+ float* c = reinterpret_cast<float*>(&frag_c);
104
+ asm volatile(
105
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
106
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
107
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
108
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
109
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
110
}
111
112
// Instruction for loading a full 16x16 matrix fragment of operand A from shared
113
// memory, directly in tensor core layout.
105
-__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
106
- uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
114
+__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
115
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
116
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
117
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
118
: "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
@@ -113,7 +122,8 @@ __device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
122
// Lookup-table based 3-input logical operation; explicitly used for
123
// dequantization as the compiler does not seem to automatically recognize it in
124
// all cases.
116
-template <int lut> __device__ inline int lop3(int a, int b, int c) {
125
+template <int lut>
126
+__device__ inline int lop3(int a, int b, int c) {
127
int res;
128
asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
129
: "=r"(res)
@@ -138,24 +148,24 @@ __device__ inline FragB dequant(int q) {
148
const int MUL = 0x2c002c00;
149
const int ADD = 0xd480d480;
150
FragB frag_b;
141
- frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
142
- *reinterpret_cast<const half2 *>(&SUB));
143
- frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
144
- *reinterpret_cast<const half2 *>(&MUL),
145
- *reinterpret_cast<const half2 *>(&ADD));
151
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152
+ *reinterpret_cast<const half2*>(&SUB));
153
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
154
+ *reinterpret_cast<const half2*>(&MUL),
155
+ *reinterpret_cast<const half2*>(&ADD));
156
return frag_b;
157
}
158
159
// Multiply dequantized values by the corresponding quantization scale; used
160
// only for grouped quantization.
151
-__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
152
- half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
161
+__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
162
+ half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
163
frag_b[0] = __hmul2(frag_b[0], s);
164
frag_b[1] = __hmul2(frag_b[1], s);
165
}
166
167
// Wait until barrier reaches `count`, then lock for current threadblock.
158
-__device__ inline void barrier_acquire(int *lock, int count) {
168
+__device__ inline void barrier_acquire(int* lock, int count) {
169
if (threadIdx.x == 0) {
170
int state = -1;
171
do
@@ -170,7 +180,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
180
}
181
182
// Release barrier and increment visitation count.
173
-__device__ inline void barrier_release(int *lock, bool reset = false) {
183
+__device__ inline void barrier_release(int* lock, bool reset = false) {
184
__syncthreads();
185
if (threadIdx.x == 0) {
186
if (reset) {
@@ -187,26 +197,27 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
197
}
198
}
199
190
-template <const int threads, // number of threads in a threadblock
191
- const int thread_m_blocks, // number of 16x16 blocks in the m
192
- // dimension (batchsize) of the threadblock
193
- const int thread_n_blocks, // same for n dimension (output)
194
- const int thread_k_blocks, // same for k dimension (reduction)
195
- const int stages, // number of stages for the async global->shared
196
- // fetch pipeline
197
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
198
- // a separate quantization scale
200
+template <const int threads, // number of threads in a threadblock
201
+ const int thread_m_blocks, // number of 16x16 blocks in the m
202
+ // dimension (batchsize) of the
203
+ // threadblock
204
+ const int thread_n_blocks, // same for n dimension (output)
205
+ const int thread_k_blocks, // same for k dimension (reduction)
206
+ const int stages, // number of stages for the async global->shared
207
+ // fetch pipeline
208
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
209
+ // with a separate quantization scale
210
>
200
-__global__ void
201
-Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
202
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
203
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
204
- const int4
205
- *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
206
- int prob_m, // batch dimension m
207
- int prob_n, // output dimension n
208
- int prob_k, // reduction dimension k
209
- int *locks // extra global storage for barrier synchronization
211
+__global__ void Marlin(
212
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
213
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
214
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
215
+ const int4* __restrict__ s, // fp16 quantization scales of shape
216
+ // (k/groupsize)xn
217
+ int prob_m, // batch dimension m
218
+ int prob_n, // output dimension n
219
+ int prob_k, // reduction dimension k
220
+ int* locks // extra global storage for barrier synchronization
221
) {
222
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
223
// same size, which might involve multiple column "slices" (of width 16 *
@@ -241,11 +252,11 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
252
int slice_row = (iters * blockIdx.x) % k_tiles;
253
int slice_col_par = (iters * blockIdx.x) / k_tiles;
254
int slice_col = slice_col_par;
244
- int slice_iters; // number of threadblock tiles in the current slice
255
+ int slice_iters; // number of threadblock tiles in the current slice
256
int slice_count =
246
- 0; // total number of active threadblocks in the current slice
247
- int slice_idx; // index of threadblock in current slice; numbered bottom to
248
- // top
257
+ 0; // total number of active threadblocks in the current slice
258
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
259
+ // top
260
261
// We can easily implement parallel problem execution by just remapping
262
// indices and advancing global pointers
@@ -261,27 +272,22 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
272
auto init_slice = [&]() {
273
slice_iters =
274
iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
264
- if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
265
- slice_iters = 0;
266
- if (slice_iters == 0)
267
- return;
268
- if (slice_row + slice_iters > k_tiles)
269
- slice_iters = k_tiles - slice_row;
275
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
276
+ if (slice_iters == 0) return;
277
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
278
slice_count = 1;
279
slice_idx = 0;
280
int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
281
if (col_first <= k_tiles * (slice_col_par + 1)) {
282
int col_off = col_first - k_tiles * slice_col_par;
283
slice_count = ceildiv(k_tiles - col_off, iters);
276
- if (col_off > 0)
277
- slice_count++;
284
+ if (col_off > 0) slice_count++;
285
int delta_first = iters * blockIdx.x - col_first;
286
if (delta_first < 0 || (col_off == 0 && delta_first == 0))
287
slice_idx = slice_count - 1;
288
else {
289
slice_idx = slice_count - 1 - delta_first / iters;
283
- if (col_off > 0)
284
- slice_idx--;
290
+ if (col_off > 0) slice_idx--;
291
}
292
}
293
if (slice_col == n_tiles) {
@@ -293,29 +299,30 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
299
};
300
init_slice();
301
296
- int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
302
+ int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
303
// We typically use `constexpr` to indicate that this value is a compile-time
304
// constant
305
constexpr int a_sh_stride =
300
- 16 * thread_k_blocks / 8; // stride of an A matrix tile in shared memory
306
+ 16 * thread_k_blocks / 8; // stride of an A matrix tile in shared memory
307
constexpr int a_gl_rd_delta_o =
308
16 * thread_k_blocks /
303
- 8; // delta between subsequent A tiles in global memory
309
+ 8; // delta between subsequent A tiles in global memory
310
int a_gl_rd_delta_i =
311
a_gl_stride *
306
- (threads / a_gl_rd_delta_o); // between subsequent accesses within a tile
312
+ (threads / a_gl_rd_delta_o); // between subsequent accesses within a tile
313
constexpr int a_sh_wr_delta =
308
- a_sh_stride * (threads / a_gl_rd_delta_o); // between shared memory writes
314
+ a_sh_stride *
315
+ (threads / a_gl_rd_delta_o); // between shared memory writes
316
constexpr int a_sh_rd_delta_o =
317
2 * ((threads / 32) /
311
- (thread_n_blocks / 4)); // between shared memory tile reads
318
+ (thread_n_blocks / 4)); // between shared memory tile reads
319
constexpr int a_sh_rd_delta_i =
313
- a_sh_stride * 16; // within a shared memory tile
320
+ a_sh_stride * 16; // within a shared memory tile
321
constexpr int a_sh_stage =
315
- a_sh_stride * (16 * thread_m_blocks); // overall size of a tile
322
+ a_sh_stride * (16 * thread_m_blocks); // overall size of a tile
323
constexpr int a_sh_wr_iters =
324
ceildiv(a_sh_stage,
318
- a_sh_wr_delta); // number of shared write iterations for a tile
325
+ a_sh_wr_delta); // number of shared write iterations for a tile
326
327
int b_gl_stride = 16 * prob_n / 32;
328
constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
@@ -368,7 +375,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
375
// needed if there are more threads than required for a certain tilesize or
376
// when the batchsize is not a multiple of 16.
377
bool a_sh_wr_pred[a_sh_wr_iters];
371
-#pragma unroll
378
+ #pragma unroll
379
for (int i = 0; i < a_sh_wr_iters; i++)
380
a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
381
bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
@@ -387,13 +394,13 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
394
// loop unrolls, all shared memory accesses are static, we simply precompute
395
// both transformed reads and writes.
396
int a_sh_wr_trans[a_sh_wr_iters];
390
-#pragma unroll
397
+ #pragma unroll
398
for (int i = 0; i < a_sh_wr_iters; i++)
399
a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
400
int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
394
-#pragma unroll
401
+ #pragma unroll
402
for (int i = 0; i < b_sh_wr_iters; i++) {
396
-#pragma unroll
403
+ #pragma unroll
404
for (int j = 0; j < thread_m_blocks; j++)
405
a_sh_rd_trans[i][j] =
406
transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -403,16 +410,16 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
410
// runtime; we break dependencies between subsequent accesses with a tile by
411
// maintining multiple pointers (we have enough registers), a tiny
412
// optimization.
406
- const int4 *B_ptr[b_sh_wr_iters];
407
-#pragma unroll
413
+ const int4* B_ptr[b_sh_wr_iters];
414
+ #pragma unroll
415
for (int i = 0; i < b_sh_wr_iters; i++)
416
B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
417
418
extern __shared__ int4 sh[];
419
// Shared memory storage for global fetch pipelines.
413
- int4 *sh_a = sh;
414
- int4 *sh_b = sh_a + (stages * a_sh_stage);
415
- int4 *sh_s = sh_b + (stages * b_sh_stage);
420
+ int4* sh_a = sh;
421
+ int4* sh_b = sh_a + (stages * a_sh_stage);
422
+ int4* sh_s = sh_b + (stages * b_sh_stage);
423
// Register storage for double buffer of shared memory reads.
424
FragA frag_a[2][thread_m_blocks];
425
I4 frag_b_quant[2];
@@ -421,34 +428,33 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
428
429
// Zero accumulators.
430
auto zero_accums = [&]() {
424
-#pragma unroll
431
+ #pragma unroll
432
for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
426
- reinterpret_cast<float *>(frag_c)[i] = 0;
433
+ reinterpret_cast<float*>(frag_c)[i] = 0;
434
};
435
436
// Asynchronously fetch the next A, B and s tile from global to the next
437
// shared memory pipeline location.
438
auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
439
if (pred) {
433
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
434
-#pragma unroll
440
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
441
+ #pragma unroll
442
for (int i = 0; i < a_sh_wr_iters; i++) {
443
cp_async4_pred(
444
&sh_a_stage[a_sh_wr_trans[i]],
445
&A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
446
a_sh_wr_pred[i]);
447
}
441
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
442
-#pragma unroll
448
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
449
+ #pragma unroll
450
for (int i = 0; i < b_sh_wr_iters; i++) {
451
cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
452
B_ptr[i] += b_gl_rd_delta_o;
453
}
454
// Only fetch scales if this tile starts a new group
455
if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
449
- int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
450
- if (s_sh_wr_pred)
451
- cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
456
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
457
+ if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
458
s_gl_rd += s_gl_rd_delta;
459
}
460
}
@@ -475,37 +481,35 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
481
// theoretically better attempts have lead to bad instruction ordering by
482
// the compiler and correspondingly a noticeable drop in performance.
483
if (group_blocks != -1) {
478
- int4 *sh_s_stage =
484
+ int4* sh_s_stage =
485
sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
486
(pipe / (group_blocks / thread_k_blocks)));
481
- reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
487
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
488
}
483
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
484
-#pragma unroll
489
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
490
+ #pragma unroll
491
for (int i = 0; i < thread_m_blocks; i++)
492
ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
487
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
488
- frag_b_quant[k % 2] = *reinterpret_cast<I4 *>(
493
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
494
+ frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
495
&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
496
};
497
498
// Execute the actual tensor core matmul of a sub-tile.
499
auto matmul = [&](int k) {
494
-// We have the m dimension as the inner loop in order to encourage overlapping
495
-// dequantization and matmul operations.
496
-#pragma unroll
500
+ // We have the m dimension as the inner loop in order to encourage overlapping
501
+ // dequantization and matmul operations.
502
+ #pragma unroll
503
for (int j = 0; j < 4; j++) {
504
int b_quant = frag_b_quant[k % 2][j];
505
int b_quant_shift = b_quant >> 8;
506
FragB frag_b0 = dequant(b_quant);
507
// If there are no groups, we can just scale the final output once and can
508
// avoid doing so for each weight.
503
- if (group_blocks != -1)
504
- scale(frag_b0, frag_s[k % 2][j], 0);
509
+ if (group_blocks != -1) scale(frag_b0, frag_s[k % 2][j], 0);
510
FragB frag_b1 = dequant(b_quant_shift);
506
- if (group_blocks != -1)
507
- scale(frag_b1, frag_s[k % 2][j], 1);
508
-#pragma unroll
511
+ if (group_blocks != -1) scale(frag_b1, frag_s[k % 2][j], 1);
512
+ #pragma unroll
513
for (int i = 0; i < thread_m_blocks; i++) {
514
mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
515
mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
@@ -530,38 +534,38 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
534
// unnecessary read or write iterations, e.g., for two warps we write only
535
// once by warp 1 and read only once by warp 0.
536
533
-#pragma unroll
537
+ #pragma unroll
538
for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
535
-#pragma unroll
539
+ #pragma unroll
540
for (int i = red_off; i > 0; i /= 2) {
541
if (i <= red_idx && red_idx < 2 * i) {
538
-#pragma unroll
542
+ #pragma unroll
543
for (int j = 0; j < 4 * 2; j++) {
544
int red_sh_wr =
545
red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
546
if (i < red_off) {
543
- float *c_rd = reinterpret_cast<float *>(
544
- &sh[red_sh_delta * j + red_sh_rd]);
545
- float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
546
-#pragma unroll
547
+ float* c_rd =
548
+ reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
549
+ float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
550
+ #pragma unroll
551
for (int k = 0; k < 4; k++)
548
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
552
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
553
c_rd[k] + c_wr[k];
554
}
555
sh[red_sh_wr] =
552
- reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
556
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
557
}
558
}
559
__syncthreads();
560
}
561
if (red_idx == 0) {
558
-#pragma unroll
562
+ #pragma unroll
563
for (int i = 0; i < 4 * 2; i++) {
560
- float *c_rd =
561
- reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
562
-#pragma unroll
564
+ float* c_rd =
565
+ reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
566
+ #pragma unroll
567
for (int j = 0; j < 4; j++)
564
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
568
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
569
c_rd[j];
570
}
571
}
@@ -571,9 +575,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
575
};
576
577
// Since multiple threadblocks may process parts of the same column slice, we
574
- // finally have to globally reduce over the results. As the striped partitioning
575
- // minimizes the number of such reductions and our outputs are usually rather
576
- // small, we perform this reduction serially in L2 cache.
578
+ // finally have to globally reduce over the results. As the striped
579
+ // partitioning minimizes the number of such reductions and our outputs are
580
+ // usually rather small, we perform this reduction serially in L2 cache.
581
auto global_reduce = [&](bool first = false, bool last = false) {
582
// We are very careful here to reduce directly in the output buffer to
583
// maximize L2 cache utilization in this step. To do this, we write out
@@ -592,39 +596,39 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
596
int row = (threadIdx.x % 32) / 4;
597
598
if (!first) {
595
-// Interestingly, doing direct global accesses here really seems to mess up the
596
-// compiler and lead to slowdowns, hence we also use async-copies even though
597
-// these fetches are not actually asynchronous.
598
-#pragma unroll
599
+ // Interestingly, doing direct global accesses here really seems to mess up
600
+ // the compiler and lead to slowdowns, hence we also use async-copies even
601
+ // though these fetches are not actually asynchronous.
602
+ #pragma unroll
603
for (int i = 0; i < thread_m_blocks * 4; i++) {
600
- cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
601
- &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
602
- c_gl_wr_delta_i * (i % 2)],
603
- i < (thread_m_blocks - 1) * 4 ||
604
- 8 * (i / 2) + row < prob_m);
604
+ cp_async4_pred(
605
+ &sh[c_sh_wr + c_sh_wr_delta * i],
606
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
607
+ c_gl_wr_delta_i * (i % 2)],
608
+ i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
609
}
610
cp_async_fence();
611
cp_async_wait<0>();
612
}
613
610
-#pragma unroll
614
+ #pragma unroll
615
for (int i = 0; i < thread_m_blocks * 4; i++) {
616
if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
617
if (!first) {
618
int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
615
-#pragma unroll
619
+ #pragma unroll
620
for (int j = 0; j < 2 * 4; j++) {
617
- reinterpret_cast<float *>(
621
+ reinterpret_cast<float*>(
622
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
619
- __half2float(reinterpret_cast<__half *>(&c_red)[j]);
623
+ __half2float(reinterpret_cast<__half*>(&c_red)[j]);
624
}
625
}
626
if (!last) {
627
int4 c;
624
-#pragma unroll
628
+ #pragma unroll
629
for (int j = 0; j < 2 * 4; j++) {
626
- reinterpret_cast<__half *>(&c)[j] =
627
- __float2half(reinterpret_cast<float *>(
630
+ reinterpret_cast<__half*>(&c)[j] =
631
+ __float2half(reinterpret_cast<float*>(
632
&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
633
}
634
C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
@@ -658,17 +662,17 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
662
663
// We first reorder in shared memory to guarantee the most efficient final
664
// global write patterns
661
- auto write = [&](int idx, float c0, float c1, FragS &s) {
665
+ auto write = [&](int idx, float c0, float c1, FragS& s) {
666
half2 res = __halves2half2(__float2half(c0), __float2half(c1));
667
if (group_blocks ==
664
- -1) // for per-column quantization we finally apply the scale here
668
+ -1) // for per-column quantization we finally apply the scale here
669
res = __hmul2(res, s[0]);
666
- ((half2 *)sh)[idx] = res;
670
+ ((half2*)sh)[idx] = res;
671
};
672
if (threadIdx.x / 32 < thread_n_blocks / 4) {
669
-#pragma unroll
673
+ #pragma unroll
674
for (int i = 0; i < thread_m_blocks; i++) {
671
-#pragma unroll
675
+ #pragma unroll
676
for (int j = 0; j < 4; j++) {
677
int wr = c_sh_wr + 8 * j;
678
write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
@@ -685,7 +689,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
689
}
690
__syncthreads();
691
688
-#pragma unroll
692
+ #pragma unroll
693
for (int i = 0;
694
i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
695
i++) {
@@ -699,9 +703,8 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
703
704
// Start global fetch and register load pipelines.
705
auto start_pipes = [&]() {
702
-#pragma unroll
703
- for (int i = 0; i < stages - 1; i++)
704
- fetch_to_shared(i, i, i < slice_iters);
706
+ #pragma unroll
707
+ for (int i = 0; i < stages - 1; i++) fetch_to_shared(i, i, i < slice_iters);
708
zero_accums();
709
wait_for_stage();
710
fetch_to_registers(0, 0);
@@ -711,12 +714,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
714
715
// Main loop.
716
while (slice_iters) {
714
-// We unroll over both the global fetch and the register load pipeline to ensure
715
-// all shared memory accesses are static. Note that both pipelines have even
716
-// length meaning that the next iteration will always start at index 0.
717
-#pragma unroll
717
+ // We unroll over both the global fetch and the register load pipeline to
718
+ // ensure all shared memory accesses are static. Note that both pipelines have
719
+ // even length meaning that the next iteration will always start at index 0.
720
+ #pragma unroll
721
for (int pipe = 0; pipe < stages;) {
719
-#pragma unroll
722
+ #pragma unroll
723
for (int k = 0; k < b_sh_wr_iters; k++) {
724
fetch_to_registers(k + 1, pipe % stages);
725
if (k == b_sh_wr_iters - 2) {
@@ -728,8 +731,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
731
matmul(k);
732
}
733
slice_iters--;
731
- if (slice_iters == 0)
732
- break;
734
+ if (slice_iters == 0) break;
735
}
736
a_gl_rd += a_gl_rd_delta_o * stages;
737
@@ -742,8 +744,7 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
744
// For per-column scales, we only fetch them here in the final step before
745
// write-out
746
if (group_blocks == -1 && last) {
745
- if (s_sh_wr_pred)
746
- cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
747
+ if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
748
cp_async_fence();
749
}
750
thread_block_reduce();
@@ -751,17 +752,17 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
752
cp_async_wait<0>();
753
__syncthreads();
754
if (threadIdx.x / 32 < thread_n_blocks / 4) {
754
- reinterpret_cast<int4 *>(&frag_s)[0] = sh_s[s_sh_rd + 0];
755
- reinterpret_cast<int4 *>(&frag_s)[1] = sh_s[s_sh_rd + 4];
755
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
756
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
757
}
758
}
758
- if (slice_count > 1) { // only globally reduce if there is more than one
759
- // block in a slice
759
+ if (slice_count > 1) { // only globally reduce if there is more than one
760
+ // block in a slice
761
barrier_acquire(&locks[slice_col], slice_idx);
762
global_reduce(slice_idx == 0, last);
763
barrier_release(&locks[slice_col], last);
764
}
764
- if (last) // only the last block in a slice actually writes the result
765
+ if (last) // only the last block in a slice actually writes the result
766
write_result();
767
slice_row = 0;
768
slice_col_par++;
@@ -770,13 +771,12 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
771
if (slice_iters) {
772
a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
773
(threadIdx.x % a_gl_rd_delta_o);
773
-#pragma unroll
774
+ #pragma unroll
775
for (int i = 0; i < b_sh_wr_iters; i++)
776
B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
777
if (slice_col == 0) {
777
-#pragma unroll
778
- for (int i = 0; i < b_sh_wr_iters; i++)
779
- B_ptr[i] -= b_gl_stride;
778
+ #pragma unroll
779
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
780
}
781
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
782
start_pipes();
@@ -787,26 +787,27 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
787
788
#else
789
790
-template <const int threads, // number of threads in a threadblock
791
- const int thread_m_blocks, // number of 16x16 blocks in the m
792
- // dimension (batchsize) of the threadblock
793
- const int thread_n_blocks, // same for n dimension (output)
794
- const int thread_k_blocks, // same for k dimension (reduction)
795
- const int stages, // number of stages for the async global->shared
796
- // fetch pipeline
797
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
798
- // a separate quantization scale
790
+template <const int threads, // number of threads in a threadblock
791
+ const int thread_m_blocks, // number of 16x16 blocks in the m
792
+ // dimension (batchsize) of the
793
+ // threadblock
794
+ const int thread_n_blocks, // same for n dimension (output)
795
+ const int thread_k_blocks, // same for k dimension (reduction)
796
+ const int stages, // number of stages for the async global->shared
797
+ // fetch pipeline
798
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
799
+ // with a separate quantization scale
800
>
800
-__global__ void
801
-Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
802
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
803
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
804
- const int4
805
- *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
806
- int prob_m, // batch dimension m
807
- int prob_n, // output dimension n
808
- int prob_k, // reduction dimension k
809
- int *locks // extra global storage for barrier synchronization
801
+__global__ void Marlin(
802
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
803
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
804
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
805
+ const int4* __restrict__ s, // fp16 quantization scales of shape
806
+ // (k/groupsize)xn
807
+ int prob_m, // batch dimension m
808
+ int prob_n, // output dimension n
809
+ int prob_k, // reduction dimension k
810
+ int* locks // extra global storage for barrier synchronization
811
) {
812
// Marlin is not implemented yet for SM < 8.0
813
assert(false);
@@ -819,10 +820,10 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
820
// than 1 warp per schedule allows some more latency hiding. At the same time,
821
// we want relatively few warps to have many registers per warp and small tiles.
822
const int USER_THREADS =
822
- 256; // Note: This is only used with user-provided thread_k/n
823
-const int STAGES = 4; // 4 pipeline stages fit into shared memory
823
+ 256; // Note: This is only used with user-provided thread_k/n
824
+const int STAGES = 4; // 4 pipeline stages fit into shared memory
825
const int SHARED_MEM =
825
- 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
826
+ 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
827
828
static constexpr int min_thread_n = 64;
829
static constexpr int min_thread_k = 64;
@@ -831,7 +832,7 @@ static constexpr int tile_size = 16;
832
static constexpr int max_par = 16;
833
834
static constexpr int pack_factor_4bit =
834
- 8; // We have 8 4-bit vals inside a 32 bit
835
+ 8; // We have 8 4-bit vals inside a 32 bit
836
837
#define __CALL_IF(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
838
GROUP_BLOCKS, NUM_THREADS) \
@@ -858,23 +859,23 @@ thread_config_t small_batch_thread_configs[] = {
859
// Ordered by priority
860
861
// thread_k, thread_n, num_threads
861
- {128, 128, 256}, // Default
862
- {128, 64, 128}, // Reduce N 2X, same K
863
- {64, 256, 256}, // Reduce K 2X, increase N 2X
864
- {64, 128, 128}, // Reduce K 2X, same N
862
+ {128, 128, 256}, // Default
863
+ {128, 64, 128}, // Reduce N 2X, same K
864
+ {64, 256, 256}, // Reduce K 2X, increase N 2X
865
+ {64, 128, 128}, // Reduce K 2X, same N
866
};
867
868
thread_config_t large_batch_thread_configs[] = {
869
// Ordered by priority
870
871
// thread_k, thread_n, num_threads
871
- {64, 256, 256}, // Default
872
- {128, 128, 256}, // Reduce N 2X, increase K 2X
873
- {64, 128, 128}, // Reduce N 2X, same K
874
- {128, 64, 128}, // Reduce N 4X, increase K 2X
872
+ {64, 256, 256}, // Default
873
+ {128, 128, 256}, // Reduce N 2X, increase K 2X
874
+ {64, 128, 128}, // Reduce N 2X, same K
875
+ {128, 64, 128}, // Reduce N 4X, increase K 2X
876
};
877
877
-bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
878
+bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
879
int prob_k) {
880
// Sanity
881
if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
@@ -907,7 +908,6 @@ bool is_valid_config(thread_config_t const &th_config, int prob_m, int prob_n,
908
}
909
910
thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
910
-
911
if (prob_m <= 16) {
912
for (auto th_config : small_batch_thread_configs) {
913
if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
@@ -926,20 +926,20 @@ thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
926
return thread_config_t{-1, -1, -1};
927
}
928
929
-#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
930
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
931
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
932
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
933
- __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
934
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
935
- __CALL_IF(2, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
936
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
937
- __CALL_IF(3, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
938
- __CALL_IF(4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
929
+#define CALL_IF(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
930
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
931
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
932
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
933
+ __CALL_IF(1, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
934
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
935
+ __CALL_IF(2, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
936
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
937
+ __CALL_IF(3, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS) \
938
+ __CALL_IF(4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
939
__CALL_IF(4, N_BLOCKS, K_BLOCKS, 8, NUM_THREADS)
940
941
-void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
942
- int prob_n, int prob_k, void *workspace, int groupsize = -1,
941
+void marlin_cuda(const void* A, const void* B, void* C, void* s, int prob_m,
942
+ int prob_n, int prob_k, void* workspace, int groupsize = -1,
943
int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
944
int thread_n = -1, int sms = -1, int max_par = 16) {
945
int tot_m = prob_m;
@@ -996,12 +996,12 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
996
" is not divisible by group_blocks = ", group_blocks);
997
}
998
999
- const int4 *A_ptr = (const int4 *)A;
1000
- const int4 *B_ptr = (const int4 *)B;
1001
- int4 *C_ptr = (int4 *)C;
1002
- const int4 *s_ptr = (const int4 *)s;
999
+ const int4* A_ptr = (const int4*)A;
1000
+ const int4* B_ptr = (const int4*)B;
1001
+ int4* C_ptr = (int4*)C;
1002
+ const int4* s_ptr = (const int4*)s;
1003
1004
- int *locks = (int *)workspace;
1004
+ int* locks = (int*)workspace;
1005
1006
for (int i = 0; i < tot_m_blocks; i += 4) {
1007
int thread_m_blocks = tot_m_blocks - i;
@@ -1011,8 +1011,7 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
1011
// Note that parallel > 1 currently only works for inputs without any
1012
// padding
1013
par = (16 * thread_m_blocks - pad) / 64;
1014
- if (par > max_par)
1015
- par = max_par;
1014
+ if (par > max_par) par = max_par;
1015
prob_m = 64 * par;
1016
i += 4 * (par - 1);
1017
thread_m_blocks = 4;
@@ -1041,12 +1040,11 @@ void marlin_cuda(const void *A, const void *B, void *C, void *s, int prob_m,
1040
}
1041
}
1042
1044
-} // namespace marlin
1043
+} // namespace marlin
1044
1046
-torch::Tensor marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1047
- torch::Tensor &b_scales, torch::Tensor &workspace,
1045
+torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1046
+ torch::Tensor& b_scales, torch::Tensor& workspace,
1047
int64_t size_m, int64_t size_n, int64_t size_k) {
1049
-
1048
// Verify M
1049
TORCH_CHECK(size_m == a.size(0),
1050
"Shape mismatch: a.size(0) = " + str(a.size(0)) +
@@ -1074,9 +1072,9 @@ torch::Tensor marlin_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1072
1073
int actual_size_n =
1074
(b_q_weight.size(1) / marlin::tile_size) * marlin::pack_factor_4bit;
1077
- TORCH_CHECK(size_n == actual_size_n,
1078
- "size_n = " + str(size_n) +
1079
- ", actual_size_n = " + str(actual_size_n));
1075
+ TORCH_CHECK(
1076
+ size_n == actual_size_n,
1077
+ "size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
1078
1079
// Verify A device and strides
1080
TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
csrc/quantization/marlin/sparse/common/base.h
+7
-5
@@ -26,12 +26,14 @@ constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
26
// corresponding index accesses must be compile-time constants, which is why we
27
// extensively use `#pragma unroll` throughout the kernel code to guarantee
28
// this.
29
-template <typename T, int n> struct Vec {
29
+template <typename T, int n>
30
+struct Vec {
31
T elems[n];
31
- __device__ T &operator[](int i) { return elems[i]; }
32
+ __device__ T& operator[](int i) { return elems[i]; }
33
};
34
34
-template <int M_, int N_, int K_> struct ShapeBase {
35
+template <int M_, int N_, int K_>
36
+struct ShapeBase {
37
static constexpr int M = M_, N = N_, K = K_;
38
};
39
@@ -44,6 +46,6 @@ using FragA = Vec<half2, 4>;
46
using FragB = Vec<half2, 2>;
47
using FragM = Vec<uint, 1>;
48
using FragC = Vec<float, 4>;
47
-using FragS = Vec<half2, 1>; // quantization scales
49
+using FragS = Vec<half2, 1>; // quantization scales
50
49
-} // namespace marlin_24
51
+} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mem.h
+34
-30
@@ -21,41 +21,44 @@
21
namespace marlin_24 {
22
// Predicated asynchronous global->shared copy; used for inputs A where we apply
23
// predication to handle batchsizes that are not multiples of 16.
24
-__device__ inline void cp_async4_pred_zfill(void *smem_ptr,
25
- const void *glob_ptr,
24
+__device__ inline void cp_async4_pred_zfill(void* smem_ptr,
25
+ const void* glob_ptr,
26
bool pred = true,
27
const bool zfill = false) {
28
const int BYTES = 16;
29
int src_in_bytes = (zfill ? 0 : BYTES);
30
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
31
- asm volatile("{\n"
32
- " .reg .pred p;\n"
33
- " setp.ne.b32 p, %0, 0;\n"
34
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
35
- "}\n" ::"r"((int)pred),
36
- "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
31
+ asm volatile(
32
+ "{\n"
33
+ " .reg .pred p;\n"
34
+ " setp.ne.b32 p, %0, 0;\n"
35
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
36
+ "}\n" ::"r"((int)pred),
37
+ "r"(smem), "l"(glob_ptr), "n"(BYTES), "r"(src_in_bytes));
38
}
39
39
-__device__ inline void cp_async4_pred(void *smem_ptr, const void *glob_ptr,
40
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
41
bool pred = true) {
42
const int BYTES = 16;
43
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
43
- asm volatile("{\n"
44
- " .reg .pred p;\n"
45
- " setp.ne.b32 p, %0, 0;\n"
46
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
47
- "}\n" ::"r"((int)pred),
48
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
44
+ asm volatile(
45
+ "{\n"
46
+ " .reg .pred p;\n"
47
+ " setp.ne.b32 p, %0, 0;\n"
48
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
49
+ "}\n" ::"r"((int)pred),
50
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
51
}
52
53
// Asynchronous global->shared copy
52
-__device__ inline void cp_async4(void *smem_ptr, const void *glob_ptr) {
54
+__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
55
const int BYTES = 16;
56
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
55
- asm volatile("{\n"
56
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
57
- "}\n" ::"r"(smem),
58
- "l"(glob_ptr), "n"(BYTES));
57
+ asm volatile(
58
+ "{\n"
59
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
60
+ "}\n" ::"r"(smem),
61
+ "l"(glob_ptr), "n"(BYTES));
62
}
63
64
// Async copy fence.
@@ -64,22 +67,23 @@ __device__ inline void cp_async_fence() {
67
}
68
69
// Wait until at most `n` async copy stages are still pending.
67
-template <int n> __device__ inline void cp_async_wait() {
70
+template <int n>
71
+__device__ inline void cp_async_wait() {
72
asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
73
}
74
75
// Instruction for loading a full 16x16 matrix fragment of operand A from shared
76
// memory, directly in tensor core layout.
73
-__device__ inline void ldsm4(FragA &frag_a, const void *smem_ptr) {
74
- uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
77
+__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
78
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
79
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
80
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
81
: "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
82
: "r"(smem));
83
}
84
81
-__device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
82
- uint32_t *a = reinterpret_cast<uint32_t *>(&frag_m);
85
+__device__ inline void ldsm4_m(FragM& frag_m, const void* smem_ptr) {
86
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_m);
87
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
88
asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n"
89
: "=r"(a[0]), "=r"(a[1])
@@ -88,8 +92,8 @@ __device__ inline void ldsm4_m(FragM &frag_m, const void *smem_ptr) {
92
93
// Instruction for loading a full 16x16 matrix fragment of operand A from shared
94
// memory, directly in tensor core layout.
91
-__device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
92
- uint32_t *a = reinterpret_cast<uint32_t *>(&frag_a);
95
+__device__ inline void ldsm4_t(FragA& frag_a, const void* smem_ptr) {
96
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
97
uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
98
asm volatile(
99
"ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
@@ -98,7 +102,7 @@ __device__ inline void ldsm4_t(FragA &frag_a, const void *smem_ptr) {
102
}
103
104
// Wait until barrier reaches `count`, then lock for current threadblock.
101
-__device__ inline void barrier_acquire(int *lock, int count) {
105
+__device__ inline void barrier_acquire(int* lock, int count) {
106
if (threadIdx.x == 0) {
107
int state = -1;
108
do
@@ -113,7 +117,7 @@ __device__ inline void barrier_acquire(int *lock, int count) {
117
}
118
119
// Release barrier and increment visitation count.
116
-__device__ inline void barrier_release(int *lock, bool reset = false) {
120
+__device__ inline void barrier_release(int* lock, bool reset = false) {
121
__syncthreads();
122
if (threadIdx.x == 0) {
123
if (reset) {
@@ -129,4 +133,4 @@ __device__ inline void barrier_release(int *lock, bool reset = false) {
133
: "l"(lock), "r"(val));
134
}
135
}
132
-} // namespace marlin_24
136
+} // namespace marlin_24
csrc/quantization/marlin/sparse/common/mma.h
+56
-51
@@ -22,51 +22,56 @@ namespace marlin_24 {
22
23
// m16n8k32 sparse tensor core mma instruction with fp16 inputs and fp32
24
// output/accumulation.
25
-__device__ inline void mma_sp(const FragB &a_frag0, const FragB &a_frag1,
26
- const FragA &frag_b, FragC &frag_c, FragM &frag_m,
25
+__device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
26
+ const FragA& frag_b, FragC& frag_c, FragM& frag_m,
27
const int psel) {
28
- const uint32_t *a0 = reinterpret_cast<const uint32_t *>(&a_frag0);
29
- const uint32_t *a1 = reinterpret_cast<const uint32_t *>(&a_frag1);
30
- const uint32_t *b = reinterpret_cast<const uint32_t *>(&frag_b);
31
- const uint32_t *e = reinterpret_cast<const uint32_t *>(&frag_m);
32
- float *c = reinterpret_cast<float *>(&frag_c);
28
+ const uint32_t* a0 = reinterpret_cast<const uint32_t*>(&a_frag0);
29
+ const uint32_t* a1 = reinterpret_cast<const uint32_t*>(&a_frag1);
30
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
31
+ const uint32_t* e = reinterpret_cast<const uint32_t*>(&frag_m);
32
+ float* c = reinterpret_cast<float*>(&frag_c);
33
if (psel == 0) {
34
- asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
35
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
36
- "{%12,%13,%14,%15}, %16, 0x0;\n"
37
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
38
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
39
- "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
40
- "f"(c[2]), "f"(c[3]), "r"(e[0]));
41
- asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
42
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
43
- "{%12,%13,%14,%15}, %16, 0x0;\n"
44
- : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
45
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
46
- "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
47
- "f"(c[6]), "f"(c[7]), "r"(e[0]));
34
+ asm volatile(
35
+ "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
36
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
37
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
38
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
39
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
40
+ "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
41
+ "r"(e[0]));
42
+ asm volatile(
43
+ "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
44
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
45
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
46
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
47
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
48
+ "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
49
+ "r"(e[0]));
50
} else {
49
- asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
50
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
51
- "{%12,%13,%14,%15}, %16, 0x1;\n"
52
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
53
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
54
- "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
55
- "f"(c[2]), "f"(c[3]), "r"(e[0]));
56
- asm volatile("mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
57
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
58
- "{%12,%13,%14,%15}, %16, 0x1;\n"
59
- : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
60
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
61
- "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
62
- "f"(c[6]), "f"(c[7]), "r"(e[0]));
51
+ asm volatile(
52
+ "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
53
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
54
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
55
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
56
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
57
+ "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
58
+ "r"(e[0]));
59
+ asm volatile(
60
+ "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
61
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
62
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
63
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
64
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
65
+ "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
66
+ "r"(e[0]));
67
}
68
}
69
70
// Lookup-table based 3-input logical operation; explicitly used for
71
// dequantization as the compiler does not seem to automatically recognize it in
72
// all cases.
69
-template <int lut> __device__ inline int lop3(int a, int b, int c) {
73
+template <int lut>
74
+__device__ inline int lop3(int a, int b, int c) {
75
int res;
76
asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
77
: "=r"(res)
@@ -120,11 +125,11 @@ __device__ inline FragB dequant_4bit(int q) {
125
const int ADD = 0xd480d480;
126
127
FragB frag_b;
123
- frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
124
- *reinterpret_cast<const half2 *>(&SUB));
125
- frag_b[1] = __hfma2(*reinterpret_cast<half2 *>(&hi),
126
- *reinterpret_cast<const half2 *>(&MUL),
127
- *reinterpret_cast<const half2 *>(&ADD));
128
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
129
+ *reinterpret_cast<const half2*>(&SUB));
130
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
131
+ *reinterpret_cast<const half2*>(&MUL),
132
+ *reinterpret_cast<const half2*>(&ADD));
133
return frag_b;
134
}
135
@@ -143,24 +148,24 @@ __device__ inline FragB dequant_8bit(int q) {
148
static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
149
150
FragB frag_b;
146
- frag_b[0] = __hsub2(*reinterpret_cast<half2 *>(&lo),
147
- *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
148
- frag_b[1] = __hsub2(*reinterpret_cast<half2 *>(&hi),
149
- *reinterpret_cast<const half2 *>(&I8s_TO_F16s_MAGIC_NUM));
151
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
153
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
154
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
155
return frag_b;
156
}
157
158
// Multiply dequantized values by the corresponding quantization scale; used
159
// only for grouped quantization.
155
-__device__ inline void scale(FragB &frag_b, FragS &frag_s, int i) {
156
- half2 s = __half2half2(reinterpret_cast<__half *>(&frag_s)[i]);
160
+__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
161
+ half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
162
frag_b[0] = __hmul2(frag_b[0], s);
163
frag_b[1] = __hmul2(frag_b[1], s);
164
}
165
161
-__device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
162
- FragS &s0, float *c4, float *c5, float *c6,
163
- float *c7, FragS &s1) {
166
+__device__ inline void scale_floats(float* c0, float* c1, float* c2, float* c3,
167
+ FragS& s0, float* c4, float* c5, float* c6,
168
+ float* c7, FragS& s1) {
169
*c0 = __fmul_rn(*c0, __half2float(s0[0].x));
170
*c1 = __fmul_rn(*c1, __half2float(s0[0].y));
171
*c2 = __fmul_rn(*c2, __half2float(s0[1].x));
@@ -172,4 +177,4 @@ __device__ inline void scale_floats(float *c0, float *c1, float *c2, float *c3,
177
*c7 = __fmul_rn(*c7, __half2float(s1[1].y));
178
}
179
175
-} // namespace marlin_24
180
+} // namespace marlin_24
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+218
-228
@@ -32,12 +32,15 @@
32
33
#else
34
35
-#include "common/mem.h"
36
-#include "common/mma.h"
35
+ #include "common/mem.h"
36
+ #include "common/mma.h"
37
38
#endif
39
40
-template <typename T> inline std::string str(T x) { return std::to_string(x); }
40
+template <typename T>
41
+inline std::string str(T x) {
42
+ return std::to_string(x);
43
+}
44
45
namespace marlin_24 {
46
@@ -45,7 +48,7 @@ namespace marlin_24 {
48
// than 1 warp per schedule allows some more latency hiding. At the same time,
49
// we want relatively few warps to have many registers per warp and small tiles.
50
static constexpr int THREADS = 256;
48
-static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
51
+static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
52
53
static constexpr int min_thread_n = 128;
54
@@ -54,35 +57,36 @@ static constexpr int max_par = 16;
57
58
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
59
57
-template <const int num_bits, // weight bits
58
- const int threads, // number of threads in a threadblock
59
- const int thread_m_blocks, // number of 16x16 blocks in the m
60
- // dimension (batchsize) of the threadblock
61
- const int thread_n_blocks, // same for n dimension (output)
62
- const int thread_k_blocks, // same for k dimension (reduction)
63
- const int stages, // number of stages for the async global->shared
64
- // fetch pipeline
65
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
66
- // a separate quantization scale
60
+template <const int num_bits, // weight bits
61
+ const int threads, // number of threads in a threadblock
62
+ const int thread_m_blocks, // number of 16x16 blocks in the m
63
+ // dimension (batchsize) of the
64
+ // threadblock
65
+ const int thread_n_blocks, // same for n dimension (output)
66
+ const int thread_k_blocks, // same for k dimension (reduction)
67
+ const int stages, // number of stages for the async global->shared
68
+ // fetch pipeline
69
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
70
+ // with a separate quantization scale
71
>
72
__global__ void Marlin_24(
69
- const int4 *__restrict__ A, // fp16 input matrix of shape mxk
70
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
71
- const int4
72
- *__restrict__ meta, // 2bit metadata information about 2:4 format on B
73
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
74
- const int4
75
- *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
76
- int prob_m, // batch dimension m
77
- int prob_n, // output dimension n
78
- int prob_k, // reduction dimension k
79
- int *locks // extra global storage for barrier synchronization
73
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
74
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
75
+ const int4* __restrict__ meta, // 2bit metadata information about 2:4
76
+ // format on B
77
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
78
+ const int4* __restrict__ s, // fp16 quantization scales of shape
79
+ // (k/groupsize)xn
80
+ int prob_m, // batch dimension m
81
+ int prob_n, // output dimension n
82
+ int prob_k, // reduction dimension k
83
+ int* locks // extra global storage for barrier synchronization
84
) {}
85
82
-torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
83
- torch::Tensor &b_meta,
84
- torch::Tensor &b_scales,
85
- torch::Tensor &workspace, int64_t num_bits,
86
+torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
87
+ torch::Tensor& b_meta,
88
+ torch::Tensor& b_scales,
89
+ torch::Tensor& workspace, int64_t num_bits,
90
int64_t size_m, int64_t size_n,
91
int64_t size_k) {
92
TORCH_CHECK_NOT_IMPLEMENTED(
@@ -92,29 +96,30 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
96
97
#else
98
95
-template <const int num_bits, // weight bits
96
- const int threads, // number of threads in a threadblock
97
- const int thread_m_blocks, // number of 16x16 blocks in the m
98
- // dimension (batchsize) of the threadblock
99
- const int thread_n_blocks, // same for n dimension (output)
100
- const int thread_k_blocks, // same for k dimension (reduction)
101
- const int stages, // number of stages for the async global->shared
102
- // fetch pipeline
103
- const int group_blocks = -1 // number of consecutive 16x16 blocks with
104
- // a separate quantization scale
99
+template <const int num_bits, // weight bits
100
+ const int threads, // number of threads in a threadblock
101
+ const int thread_m_blocks, // number of 16x16 blocks in the m
102
+ // dimension (batchsize) of the
103
+ // threadblock
104
+ const int thread_n_blocks, // same for n dimension (output)
105
+ const int thread_k_blocks, // same for k dimension (reduction)
106
+ const int stages, // number of stages for the async global->shared
107
+ // fetch pipeline
108
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
109
+ // with a separate quantization scale
110
>
111
__global__ void Marlin_24(
107
- const int4 *__restrict__ A, // fp16 input matrix of shape mxk
108
- const int4 *__restrict__ B, // 4bit quantized weight matrix of shape kxn
109
- const int4
110
- *__restrict__ meta, // 2bit metadata information about 2:4 format on B
111
- int4 *__restrict__ C, // fp16 output buffer of shape mxn
112
- const int4
113
- *__restrict__ s, // fp16 quantization scales of shape (k/groupsize)xn
114
- int prob_m, // batch dimension m
115
- int prob_n, // output dimension n
116
- int prob_k, // reduction dimension k
117
- int *locks // extra global storage for barrier synchronization
112
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
113
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
114
+ const int4* __restrict__ meta, // 2bit metadata information about 2:4
115
+ // format on B
116
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
117
+ const int4* __restrict__ s, // fp16 quantization scales of shape
118
+ // (k/groupsize)xn
119
+ int prob_m, // batch dimension m
120
+ int prob_n, // output dimension n
121
+ int prob_k, // reduction dimension k
122
+ int* locks // extra global storage for barrier synchronization
123
) {
124
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
125
// same size, which might involve multiple column "slices" (of width 16 *
@@ -174,27 +179,22 @@ __global__ void Marlin_24(
179
auto init_slice = [&]() {
180
slice_iters =
181
iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
177
- if (slice_iters < 0 || slice_col_par >= n_tiles * parallel)
178
- slice_iters = 0;
179
- if (slice_iters == 0)
180
- return;
181
- if (slice_row + slice_iters > k_tiles)
182
- slice_iters = k_tiles - slice_row;
182
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
183
+ if (slice_iters == 0) return;
184
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
185
slice_count = 1;
186
slice_idx = 0;
187
int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
188
if (col_first <= k_tiles * (slice_col_par + 1)) {
189
int col_off = col_first - k_tiles * slice_col_par;
190
slice_count = ceildiv(k_tiles - col_off, iters);
189
- if (col_off > 0)
190
- slice_count++;
191
+ if (col_off > 0) slice_count++;
192
int delta_first = iters * blockIdx.x - col_first;
193
if (delta_first < 0 || (col_off == 0 && delta_first == 0))
194
slice_idx = slice_count - 1;
195
else {
196
slice_idx = slice_count - 1 - delta_first / iters;
196
- if (col_off > 0)
197
- slice_idx--;
197
+ if (col_off > 0) slice_idx--;
198
}
199
}
200
if (slice_col == n_tiles) {
@@ -207,7 +207,7 @@ __global__ void Marlin_24(
207
init_slice();
208
209
// RLC: 8 is vec_size -> 128-bit instructions, 8 fp16 elements
210
- int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
210
+ int a_gl_stride = prob_k / 8; // stride of the A matrix in global memory
211
212
// stride of an A matrix tile in shared memory
213
constexpr int a_sh_stride = 32 * thread_k_blocks / 8;
@@ -239,9 +239,9 @@ __global__ void Marlin_24(
239
constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
240
constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
241
242
- int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
242
+ int m_gl_stride = 2 * prob_n / 8; // (16*2*4 / 8) = 16
243
constexpr int m_sh_stride =
244
- (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
244
+ (16 * thread_n_blocks) / 4; // #warps n-dim * threads/warp
245
int m_gl_rd_delta_o = m_gl_stride * thread_k_blocks;
246
int m_gl_rd_delta_i = m_gl_stride * (threads / m_sh_stride);
247
constexpr int m_sh_wr_delta = threads / 2;
@@ -305,7 +305,7 @@ __global__ void Marlin_24(
305
// needed if there are more threads than required for a certain tilesize or
306
// when the batchsize is not a multiple of 16.
307
bool a_sh_wr_pred[a_sh_wr_iters];
308
-#pragma unroll
308
+ #pragma unroll
309
for (int i = 0; i < a_sh_wr_iters; i++) {
310
a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
311
}
@@ -325,13 +325,13 @@ __global__ void Marlin_24(
325
// loop unrolls, all shared memory accesses are static, we simply precompute
326
// both transformed reads and writes.
327
int a_sh_wr_trans[a_sh_wr_iters];
328
-#pragma unroll
328
+ #pragma unroll
329
for (int i = 0; i < a_sh_wr_iters; i++)
330
a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
331
int a_sh_rd_trans[2][b_sh_wr_iters][thread_m_blocks];
332
-#pragma unroll
332
+ #pragma unroll
333
for (int i = 0; i < b_sh_wr_iters; i++) {
334
-#pragma unroll
334
+ #pragma unroll
335
for (int j = 0; j < thread_m_blocks; j++) {
336
a_sh_rd_trans[0][i][j] =
337
transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
@@ -344,23 +344,23 @@ __global__ void Marlin_24(
344
// runtime; we break dependencies between subsequent accesses with a tile by
345
// maintining multiple pointers (we have enough registers), a tiny
346
// optimization.
347
- const int4 *B_ptr[b_sh_wr_iters];
348
-#pragma unroll
347
+ const int4* B_ptr[b_sh_wr_iters];
348
+ #pragma unroll
349
for (int i = 0; i < b_sh_wr_iters; i++)
350
B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
351
352
bool m_sh_wr_pred = threadIdx.x < m_sh_wr_delta;
353
- const int4 *meta_ptr[m_sh_iters];
354
-#pragma unroll
353
+ const int4* meta_ptr[m_sh_iters];
354
+ #pragma unroll
355
for (int i = 0; i < m_sh_iters; i++)
356
meta_ptr[i] = meta + m_gl_rd_delta_i * i + m_gl_rd;
357
358
extern __shared__ int4 sh[];
359
// Shared memory storage for global fetch pipelines.
360
- int4 *sh_a = sh;
361
- int4 *sh_b = sh_a + (stages * a_sh_stage);
362
- int4 *sh_s = sh_b + (stages * b_sh_stage);
363
- int4 *sh_m = sh_s + (stages * s_sh_stage);
360
+ int4* sh_a = sh;
361
+ int4* sh_b = sh_a + (stages * a_sh_stage);
362
+ int4* sh_s = sh_b + (stages * b_sh_stage);
363
+ int4* sh_m = sh_s + (stages * s_sh_stage);
364
// Register storage for double buffer of shared memory reads.
365
FragA frag_a[2][thread_m_blocks][2];
366
I4 frag_b_quant[2][b_thread_vecs];
@@ -370,46 +370,43 @@ __global__ void Marlin_24(
370
371
// Zero accumulators.
372
auto zero_accums = [&]() {
373
-#pragma unroll
373
+ #pragma unroll
374
for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
375
- reinterpret_cast<float *>(frag_c)[i] = 0;
375
+ reinterpret_cast<float*>(frag_c)[i] = 0;
376
};
377
378
// Asynchronously fetch the next A, B and s tile from global to the next
379
// shared memory pipeline location.
380
auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
381
if (pred) {
382
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
383
-#pragma unroll
382
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
383
+ #pragma unroll
384
for (int i = 0; i < a_sh_wr_iters; i++) {
385
cp_async4_pred(
386
&sh_a_stage[a_sh_wr_trans[i]],
387
&A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
388
a_sh_wr_pred[i]);
389
}
390
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
391
-#pragma unroll
390
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
391
+ #pragma unroll
392
for (int i = 0; i < b_sh_wr_iters; i++) {
393
-#pragma unroll
393
+ #pragma unroll
394
for (int j = 0; j < b_thread_vecs; j++) {
395
- cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j],
396
- B_ptr[i] + j);
395
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
396
}
397
B_ptr[i] += b_gl_rd_delta_o;
398
}
400
- int4 *sh_meta_stage = sh_m + m_sh_stage * pipe;
401
-#pragma unroll
399
+ int4* sh_meta_stage = sh_m + m_sh_stage * pipe;
400
+ #pragma unroll
401
for (int i = 0; i < m_sh_iters; i++) {
402
if (m_sh_wr_pred)
404
- cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr],
405
- meta_ptr[i]);
403
+ cp_async4(&sh_meta_stage[m_sh_wr_delta * i + m_sh_wr], meta_ptr[i]);
404
meta_ptr[i] += m_gl_rd_delta_o;
405
}
406
// Only fetch scales if this tile starts a new group
407
if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
410
- int4 *sh_s_stage = sh_s + s_sh_stage * pipe;
411
- if (s_sh_wr_pred)
412
- cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
408
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
409
+ if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
410
s_gl_rd += s_gl_rd_delta;
411
}
412
}
@@ -436,13 +433,13 @@ __global__ void Marlin_24(
433
// theoretically better attempts have lead to bad instruction ordering by
434
// the compiler and correspondingly a noticeable drop in performance.
435
if (group_blocks != -1) {
439
- int4 *sh_s_stage =
436
+ int4* sh_s_stage =
437
sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
438
(pipe / (group_blocks / thread_k_blocks)));
442
- reinterpret_cast<int4 *>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
439
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
440
}
444
- int4 *sh_a_stage = sh_a + a_sh_stage * pipe;
445
-#pragma unroll
441
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
442
+ #pragma unroll
443
for (int i = 0; i < thread_m_blocks; i++) {
444
ldsm4(frag_a[k % 2][i][0],
445
&sh_a_stage[a_sh_rd_trans[0][k % b_sh_wr_iters][i]]);
@@ -450,24 +447,24 @@ __global__ void Marlin_24(
447
&sh_a_stage[a_sh_rd_trans[1][k % b_sh_wr_iters][i]]);
448
}
449
453
- int4 *sh_b_stage = sh_b + b_sh_stage * pipe;
454
-#pragma unroll
450
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
451
+ #pragma unroll
452
for (int i = 0; i < b_thread_vecs; i++) {
456
- frag_b_quant[k % 2][i] = *reinterpret_cast<I4 *>(
453
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
454
&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
455
}
456
457
// Load meta with ldsm4
461
- int4 *sh_m_stage = sh_m + m_sh_stage * pipe;
458
+ int4* sh_m_stage = sh_m + m_sh_stage * pipe;
459
ldsm4_m(frag_m[k % 2][0],
460
&sh_m_stage[m_sh_rd_delta * (k % m_sh_iters) + m_sh_rd]);
461
};
462
463
// Execute the actual tensor core matmul of a sub-tile.
464
auto matmul = [&](int k) {
468
-// We have the m dimension as the inner loop in order to encourage overlapping
469
-// dequantization and matmul operations.
470
-#pragma unroll
465
+ // We have the m dimension as the inner loop in order to encourage overlapping
466
+ // dequantization and matmul operations.
467
+ #pragma unroll
468
for (int j = 0; j < 4; j++) {
469
FragB frag_b0;
470
FragB frag_b1;
@@ -480,7 +477,7 @@ __global__ void Marlin_24(
477
frag_b1 = dequant_4bit(b_quant_shift);
478
479
} else {
483
- int *frag_b_quant_ptr = reinterpret_cast<int *>(frag_b_quant[k % 2]);
480
+ int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
481
int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
482
int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
483
@@ -497,7 +494,7 @@ __global__ void Marlin_24(
494
scale(frag_b1, frag_s[k % 2][j], 1);
495
}
496
500
-#pragma unroll
497
+ #pragma unroll
498
for (int i = 0; i < thread_m_blocks; i++) {
499
mma_sp(frag_b0, frag_b1, frag_a[k % 2][i][0], frag_c[i][j][0],
500
frag_m[k % 2][j / 2], j % 2);
@@ -518,41 +515,41 @@ __global__ void Marlin_24(
515
int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
516
(threadIdx.x % b_sh_stride_threads);
517
521
-// Parallel logarithmic shared memory reduction. We make sure to avoid any
522
-// unnecessary read or write iterations, e.g., for two warps we write only once
523
-// by warp 1 and read only once by warp 0.
524
-#pragma unroll
518
+ // Parallel logarithmic shared memory reduction. We make sure to avoid any
519
+ // unnecessary read or write iterations, e.g., for two warps we write only
520
+ // once by warp 1 and read only once by warp 0.
521
+ #pragma unroll
522
for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
526
-#pragma unroll
523
+ #pragma unroll
524
for (int i = red_off; i > 0; i /= 2) {
525
if (i <= red_idx && red_idx < 2 * i) {
529
-#pragma unroll
526
+ #pragma unroll
527
for (int j = 0; j < 4 * 2; j++) {
528
int red_sh_wr =
529
red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
530
if (i < red_off) {
534
- float *c_rd = reinterpret_cast<float *>(
535
- &sh[red_sh_delta * j + red_sh_rd]);
536
- float *c_wr = reinterpret_cast<float *>(&sh[red_sh_wr]);
537
-#pragma unroll
531
+ float* c_rd =
532
+ reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
533
+ float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
534
+ #pragma unroll
535
for (int k = 0; k < 4; k++)
539
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + j][k] +=
536
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
537
c_rd[k] + c_wr[k];
538
}
539
sh[red_sh_wr] =
543
- reinterpret_cast<int4 *>(&frag_c)[4 * 2 * m_block + j];
540
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
541
}
542
}
543
__syncthreads();
544
}
545
if (red_idx == 0) {
549
-#pragma unroll
546
+ #pragma unroll
547
for (int i = 0; i < 4 * 2; i++) {
551
- float *c_rd =
552
- reinterpret_cast<float *>(&sh[red_sh_delta * i + red_sh_rd]);
553
-#pragma unroll
548
+ float* c_rd =
549
+ reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
550
+ #pragma unroll
551
for (int j = 0; j < 4; j++)
555
- reinterpret_cast<FragC *>(frag_c)[4 * 2 * m_block + i][j] +=
552
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
553
c_rd[j];
554
}
555
}
@@ -562,9 +559,9 @@ __global__ void Marlin_24(
559
};
560
561
// Since multiple threadblocks may process parts of the same column slice, we
565
- // finally have to globally reduce over the results. As the striped partitioning
566
- // minimizes the number of such reductions and our outputs are usually rather
567
- // small, we perform this reduction serially in L2 cache.
562
+ // finally have to globally reduce over the results. As the striped
563
+ // partitioning minimizes the number of such reductions and our outputs are
564
+ // usually rather small, we perform this reduction serially in L2 cache.
565
auto global_reduce = [&](bool first = false, bool last = false) {
566
// We are very careful here to reduce directly in the output buffer to
567
// maximize L2 cache utilization in this step. To do this, we write out
@@ -574,7 +571,7 @@ __global__ void Marlin_24(
571
int c_gl_stride = prob_n / 8;
572
int c_gl_wr_delta_o = 2 * 4 * c_gl_stride;
573
int c_gl_wr_delta_i =
577
- c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
574
+ c_gl_stride; // 8 threads (e.g., 0,4,8,12,16,20,24,28)
575
int c_gl_wr = 2 * c_gl_stride * (threadIdx.x % 4) +
576
8 * (threadIdx.x / 32) + (threadIdx.x % 32) / 4;
577
c_gl_wr += (2 * thread_n_blocks) * slice_col;
@@ -584,10 +581,10 @@ __global__ void Marlin_24(
581
int col = 2 * ((threadIdx.x % 32) % 4);
582
583
if (!first) {
587
-// Interestingly, doing direct global accesses here really seems to mess up the
588
-// compiler and lead to slowdowns, hence we also use async-copies even though
589
-// these fetches are not actually asynchronous.
590
-#pragma unroll
584
+ // Interestingly, doing direct global accesses here really seems to mess up
585
+ // the compiler and lead to slowdowns, hence we also use async-copies even
586
+ // though these fetches are not actually asynchronous.
587
+ #pragma unroll
588
for (int i = 0; i < thread_m_blocks * 4; i++) {
589
cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i],
590
&C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
@@ -599,32 +596,32 @@ __global__ void Marlin_24(
596
cp_async_wait<0>();
597
}
598
602
-#pragma unroll
599
+ #pragma unroll
600
for (int i = 0; i < thread_m_blocks * 4; i++) {
601
if (i < (thread_m_blocks - 1) * 4 ||
602
8 * (i / 2) + col + (i % 2) < prob_m) {
603
if (!first) {
604
int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
608
-#pragma unroll
605
+ #pragma unroll
606
for (int j2 = 0; j2 < 2; j2++) {
610
-#pragma unroll
607
+ #pragma unroll
608
for (int j1 = 0; j1 < 4; j1++) {
612
- reinterpret_cast<float *>(
609
+ reinterpret_cast<float*>(
610
&frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
611
4 * ((i % 4) / 2) + i % 2] +=
612
__half2float(
616
- reinterpret_cast<__half *>(&c_red)[(j2 * 4 + j1)]);
613
+ reinterpret_cast<__half*>(&c_red)[(j2 * 4 + j1)]);
614
}
615
}
616
}
617
if (!last) {
618
int4 c;
622
-#pragma unroll
619
+ #pragma unroll
620
for (int j2 = 0; j2 < 2; j2++) {
624
-#pragma unroll
621
+ #pragma unroll
622
for (int j1 = 0; j1 < 4; j1++) {
626
- reinterpret_cast<__half *>(&c)[(j2 * 4 + j1)] =
627
- __float2half(reinterpret_cast<float *>(
623
+ reinterpret_cast<__half*>(&c)[(j2 * 4 + j1)] =
624
+ __float2half(reinterpret_cast<float*>(
625
&frag_c)[4 * 2 * 4 * (i / 4) + 8 * j1 + 2 * j2 +
626
4 * ((i % 4) / 2) + i % 2]);
627
}
@@ -643,9 +640,9 @@ __global__ void Marlin_24(
640
auto write_result = [&]() {
641
int c_gl_stride = prob_n / 8;
642
646
- constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
647
- constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
648
- constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
643
+ constexpr int c_sh_stride = 2 * thread_n_blocks; // RLC:
644
+ constexpr int c_sh_stride_2 = 2 * c_sh_stride + 2; // RLC:
645
+ constexpr int c_sh_stride_3 = 2 * (2 * thread_n_blocks) + 2; // RLC:
646
647
int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
648
@@ -654,22 +651,22 @@ __global__ void Marlin_24(
651
c_gl_wr += (2 * thread_n_blocks) * slice_col;
652
653
int c_sh_wr = c_sh_stride_2 * ((threadIdx.x % 32) % 4) +
657
- ((threadIdx.x % 32) / 4); // RLC:
658
- c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
654
+ ((threadIdx.x % 32) / 4); // RLC:
655
+ c_sh_wr += 8 * (threadIdx.x / 32); // 128/4(half4)
656
657
constexpr int c_sh_rd_delta =
661
- c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
658
+ c_sh_stride_3 * (threads / (2 * 2 * thread_n_blocks)); // RLC:
659
int c_sh_rd = c_sh_stride_3 * (threadIdx.x / (2 * 2 * thread_n_blocks)) +
660
(threadIdx.x % (2 * 2 * thread_n_blocks));
661
662
int c_gl_wr_end = c_gl_stride * prob_m;
663
667
- auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS &s0,
668
- float c4, float c5, float c6, float c7, FragS &s1) {
664
+ auto write = [&](int idx, float c0, float c1, float c2, float c3, FragS& s0,
665
+ float c4, float c5, float c6, float c7, FragS& s1) {
666
uint2 res[2];
667
res[0] = to_half4(c0, c1, c2, c3);
668
res[1] = to_half4(c4, c5, c6, c7);
672
- half2 *tmp = (half2 *)&res;
669
+ half2* tmp = (half2*)&res;
670
// for per-column quantization we finally apply the scale here
671
if constexpr (group_blocks == -1 && num_bits == 4) {
672
tmp[0] = __hmul2(tmp[0], s0[0]);
@@ -677,12 +674,12 @@ __global__ void Marlin_24(
674
tmp[2] = __hmul2(tmp[2], s1[0]);
675
tmp[3] = __hmul2(tmp[3], s1[1]);
676
}
680
- ((int4 *)sh)[idx] = *((int4 *)&res[0]);
677
+ ((int4*)sh)[idx] = *((int4*)&res[0]);
678
};
679
680
// RLC: only warp 0 and 1 baseline example
681
if (threadIdx.x / 32 < thread_n_blocks / 4) {
685
-#pragma unroll
682
+ #pragma unroll
683
for (int i = 0; i < thread_m_blocks; i++) {
684
int wr = c_sh_wr;
685
write(wr, frag_c[i][0][0][0], frag_c[i][1][0][0], frag_c[i][2][0][0],
@@ -707,7 +704,7 @@ __global__ void Marlin_24(
704
}
705
__syncthreads();
706
710
-#pragma unroll
707
+ #pragma unroll
708
for (int i = 0;
709
i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
710
i++) {
@@ -721,9 +718,8 @@ __global__ void Marlin_24(
718
719
// Start global fetch and register load pipelines.
720
auto start_pipes = [&]() {
724
-#pragma unroll
725
- for (int i = 0; i < stages - 1; i++)
726
- fetch_to_shared(i, i, i < slice_iters);
721
+ #pragma unroll
722
+ for (int i = 0; i < stages - 1; i++) fetch_to_shared(i, i, i < slice_iters);
723
zero_accums();
724
wait_for_stage();
725
fetch_to_registers(0, 0);
@@ -733,10 +729,10 @@ __global__ void Marlin_24(
729
730
// Main loop.
731
while (slice_iters) {
736
-// We unroll over both the global fetch and the register load pipeline to ensure
737
-// all shared memory accesses are static. Note that both pipelines have even
738
-// length meaning that the next iteration will always start at index 0.
739
-#pragma unroll
732
+ // We unroll over both the global fetch and the register load pipeline to
733
+ // ensure all shared memory accesses are static. Note that both pipelines have
734
+ // even length meaning that the next iteration will always start at index 0.
735
+ #pragma unroll
736
for (int pipe = 0; pipe < stages;) {
737
fetch_to_shared((pipe + stages - 1) % stages, pipe,
738
slice_iters >= stages);
@@ -747,8 +743,7 @@ __global__ void Marlin_24(
743
744
pipe++;
745
slice_iters--;
750
- if (slice_iters == 0)
751
- break;
746
+ if (slice_iters == 0) break;
747
}
748
a_gl_rd += a_gl_rd_delta_o * stages;
749
@@ -762,13 +757,11 @@ __global__ void Marlin_24(
757
// write-out
758
if constexpr (group_blocks == -1) {
759
if constexpr (num_bits == 8) {
765
- if (s_sh_wr_pred)
766
- cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
760
+ if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
761
cp_async_fence();
762
} else {
763
if (last) {
770
- if (s_sh_wr_pred)
771
- cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
764
+ if (s_sh_wr_pred) cp_async4(&sh_s[s_sh_wr], &s[s_gl_rd]);
765
cp_async_fence();
766
}
767
}
@@ -780,14 +773,14 @@ __global__ void Marlin_24(
773
cp_async_wait<0>();
774
__syncthreads();
775
if (threadIdx.x / 32 < thread_n_blocks / 4) {
783
- *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
776
+ *(float4*)(frag_s) = *(float4*)(&sh_s[s_sh_rd]);
777
}
778
} else {
779
if (last) {
780
cp_async_wait<0>();
781
__syncthreads();
782
if (threadIdx.x / 32 < thread_n_blocks / 4) {
790
- *(float4 *)(frag_s) = *(float4 *)(&sh_s[s_sh_rd]);
783
+ *(float4*)(frag_s) = *(float4*)(&sh_s[s_sh_rd]);
784
}
785
}
786
}
@@ -798,7 +791,7 @@ __global__ void Marlin_24(
791
// overflow in fp16)
792
if constexpr (group_blocks == -1 && num_bits == 8) {
793
if (threadIdx.x / 32 < thread_n_blocks / 4) {
801
-#pragma unroll
794
+ #pragma unroll
795
for (int i = 0; i < thread_m_blocks; i++) {
796
scale_floats(&frag_c[i][0][0][0], &frag_c[i][1][0][0],
797
&frag_c[i][2][0][0], &frag_c[i][3][0][0], frag_s[0][0],
@@ -827,13 +820,13 @@ __global__ void Marlin_24(
820
}
821
}
822
830
- if (slice_count > 1) { // only globally reduce if there is more than one
831
- // block in a slice
823
+ if (slice_count > 1) { // only globally reduce if there is more than one
824
+ // block in a slice
825
barrier_acquire(&locks[slice_col], slice_idx);
826
global_reduce(slice_idx == 0, last);
827
barrier_release(&locks[slice_col], last);
828
}
836
- if (last) // only the last block in a slice actually writes the result
829
+ if (last) // only the last block in a slice actually writes the result
830
write_result();
831
832
slice_row = 0;
@@ -843,19 +836,17 @@ __global__ void Marlin_24(
836
if (slice_iters) {
837
a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
838
(threadIdx.x % a_gl_rd_delta_o);
846
-#pragma unroll
839
+ #pragma unroll
840
for (int i = 0; i < b_sh_wr_iters; i++)
841
B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
849
-#pragma unroll
842
+ #pragma unroll
843
for (int i = 0; i < m_sh_iters; i++)
844
meta_ptr[i] += (m_sh_stride)-m_gl_rd_delta_o * k_tiles;
845
if (slice_col == 0) {
853
-#pragma unroll
854
- for (int i = 0; i < b_sh_wr_iters; i++)
855
- B_ptr[i] -= b_gl_stride;
856
-#pragma unroll
857
- for (int i = 0; i < m_sh_iters; i++)
858
- meta_ptr[i] -= m_gl_stride;
846
+ #pragma unroll
847
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
848
+ #pragma unroll
849
+ for (int i = 0; i < m_sh_iters; i++) meta_ptr[i] -= m_gl_stride;
850
}
851
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
852
start_pipes();
@@ -866,26 +857,26 @@ __global__ void Marlin_24(
857
858
#endif
859
869
-#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
870
- THREAD_K_BLOCKS, GROUP_BLOCKS) \
871
- else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
872
- thread_n_blocks == THREAD_N_BLOCKS && \
873
- thread_k_blocks == THREAD_K_BLOCKS && \
874
- group_blocks == GROUP_BLOCKS) { \
875
- cudaFuncSetAttribute( \
876
- Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
877
- THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
878
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
879
- Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
880
- THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
881
- <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
882
- C_ptr, s_ptr, prob_n, \
883
- prob_m, prob_k, locks); \
860
+#define CALL_IF_2_4(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
861
+ THREAD_K_BLOCKS, GROUP_BLOCKS) \
862
+ else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
863
+ thread_n_blocks == THREAD_N_BLOCKS && \
864
+ thread_k_blocks == THREAD_K_BLOCKS && \
865
+ group_blocks == GROUP_BLOCKS) { \
866
+ cudaFuncSetAttribute( \
867
+ Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
868
+ THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS>, \
869
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
870
+ Marlin_24<NUM_BITS, THREADS, THREAD_N_BLOCKS, THREAD_M_BLOCKS, \
871
+ THREAD_K_BLOCKS, STAGES, GROUP_BLOCKS> \
872
+ <<<blocks, THREADS, max_shared_mem, stream>>>(A_ptr, B_ptr, meta_ptr, \
873
+ C_ptr, s_ptr, prob_n, \
874
+ prob_m, prob_k, locks); \
875
}
876
886
-void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
887
- void *s, int prob_m, int prob_n, int prob_k,
888
- void *workspace, int num_bits, int groupsize = -1,
877
+void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
878
+ void* s, int prob_m, int prob_n, int prob_k,
879
+ void* workspace, int num_bits, int groupsize = -1,
880
int dev = 0, cudaStream_t stream = 0, int thread_k = -1,
881
int thread_m = -1, int sms = -1, int max_par = 16) {
882
int tot_n = prob_n;
@@ -904,8 +895,8 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
895
896
if (thread_k == -1 || thread_m == -1) {
897
if (prob_n <= 16) {
907
- // For small batchizes, better partitioningif is slightly more important than
908
- // better compute utilization
898
+ // For small batchizes, better partitioningif is slightly more important
899
+ // than better compute utilization
900
thread_k = 128;
901
thread_m = 128;
902
} else {
@@ -914,7 +905,7 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
905
}
906
}
907
917
- int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
908
+ int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
909
int thread_m_blocks = thread_m / 16;
910
int group_blocks = (groupsize == -1) ? -1 : groupsize / 16;
911
int blocks = sms;
@@ -931,13 +922,13 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
922
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
923
", ", prob_n, ", ", prob_k, "]");
924
934
- const int4 *A_ptr = (const int4 *)A;
935
- const int4 *B_ptr = (const int4 *)B;
936
- const int4 *meta_ptr = (const int4 *)meta;
937
- int4 *C_ptr = (int4 *)C;
938
- const int4 *s_ptr = (const int4 *)s;
925
+ const int4* A_ptr = (const int4*)A;
926
+ const int4* B_ptr = (const int4*)B;
927
+ const int4* meta_ptr = (const int4*)meta;
928
+ int4* C_ptr = (int4*)C;
929
+ const int4* s_ptr = (const int4*)s;
930
940
- int *locks = (int *)workspace;
931
+ int* locks = (int*)workspace;
932
for (int i = 0; i < tot_n_blocks; i += 4) {
933
int thread_n_blocks = tot_n_blocks - i;
934
prob_n = tot_n - 16 * i;
@@ -946,8 +937,7 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
937
// Note that parallel > 1 currently only works for inputs without any
938
// padding
939
par = (16 * thread_n_blocks - pad) / 64;
949
- if (par > max_par)
950
- par = max_par;
940
+ if (par > max_par) par = max_par;
941
prob_n = 64 * par;
942
i += 4 * (par - 1);
943
thread_n_blocks = 4;
@@ -956,16 +946,16 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
946
// For compilation speed, we only define the kernel configurations that have
947
// seemed useful (in terms of performance) in our testing, however many more
948
// are, in principle, possible.
959
-
949
+
950
// the false is start of the CALL_IF macros
961
- if (false) {
962
- } // BMxBNxBK, group
951
+ if (false) {
952
+ } // BMxBNxBK, group
953
// 4-bit
964
- CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
965
- CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
966
- CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
967
- CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
968
- CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
954
+ CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
955
+ CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
956
+ CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
957
+ CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
958
+ CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
959
CALL_IF_2_4(4, 16, 2, 2, 4)
960
CALL_IF_2_4(4, 16, 3, 2, -1)
961
CALL_IF_2_4(4, 16, 3, 2, 4)
@@ -973,11 +963,11 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
963
CALL_IF_2_4(4, 16, 4, 2, 4)
964
965
// 8-bit
976
- CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
977
- CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
978
- CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
979
- CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
980
- CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
966
+ CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
967
+ CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
968
+ CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
969
+ CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
970
+ CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
971
CALL_IF_2_4(8, 16, 2, 2, 4)
972
CALL_IF_2_4(8, 16, 3, 2, -1)
973
CALL_IF_2_4(8, 16, 3, 2, 4)
@@ -997,12 +987,12 @@ void marlin_cuda_2_4(const void *A, const void *B, const void *meta, void *C,
987
}
988
}
989
1000
-} // namespace marlin_24
990
+} // namespace marlin_24
991
1002
-torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1003
- torch::Tensor &b_meta,
1004
- torch::Tensor &b_scales,
1005
- torch::Tensor &workspace, int64_t num_bits,
992
+torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
993
+ torch::Tensor& b_meta,
994
+ torch::Tensor& b_scales,
995
+ torch::Tensor& workspace, int64_t num_bits,
996
int64_t size_m, int64_t size_n,
997
int64_t size_k) {
998
// Verify num_bits
@@ -1037,9 +1027,9 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1027
" is not divisible by tile_size = " + str(marlin_24::tile_size));
1028
1029
int actual_size_n = (b_q_weight.size(1) / marlin_24::tile_size) * pack_factor;
1040
- TORCH_CHECK(size_n == actual_size_n,
1041
- "size_n = " + str(size_n) +
1042
- ", actual_size_n = " + str(actual_size_n));
1030
+ TORCH_CHECK(
1031
+ size_n == actual_size_n,
1032
+ "size_n = " + str(size_n) + ", actual_size_n = " + str(actual_size_n));
1033
1034
// Verify meta
1035
TORCH_CHECK(b_meta.size(0) == size_k / 8 / 2 / 2,
@@ -1081,7 +1071,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor &a, torch::Tensor &b_q_weight,
1071
", is not divisible by b_scales.size(0) = " +
1072
str(b_scales.size(0)));
1073
groupsize = size_k / b_scales.size(0);
1084
- groupsize /= 2; // Because of 24
1074
+ groupsize /= 2; // Because of 24
1075
}
1076
1077
// Verify groupsize
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+27
-36
@@ -22,27 +22,23 @@ __device__ inline unsigned int as_unsigned(int i) {
22
// 4-bit matvec kernel (LUT-based)
23
__global__ void NUQ4MatMulKernel(
24
#ifndef USE_ROCM
25
- const half2* __restrict__ vec,
25
+ const half2* __restrict__ vec,
26
#else
27
- const __half2* __restrict__ vec,
27
+ const __half2* __restrict__ vec,
28
#endif
29
- const int* __restrict__ mat,
29
+ const int* __restrict__ mat,
30
#ifndef USE_ROCM
31
- half2* __restrict__ mul,
31
+ half2* __restrict__ mul,
32
#else
33
- float2* __restrict__ mul,
33
+ float2* __restrict__ mul,
34
#endif
35
- const __half* __restrict__ lookup_table,
36
- int height,
37
- int width,
38
- int batch,
39
- int vec_height
40
-) {
35
+ const __half* __restrict__ lookup_table, int height, int width, int batch,
36
+ int vec_height) {
37
38
const int blockwidth2 = BLOCKWIDTH / 2;
39
40
int row = BLOCKHEIGHT4 * blockIdx.x;
45
- int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
41
+ int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
42
43
#ifndef USE_ROCM
44
__shared__ half2 blockvec[blockwidth2];
@@ -73,14 +69,16 @@ __global__ void NUQ4MatMulKernel(
69
unsigned int tmp1;
70
unsigned int lut_index1, lut_index2;
71
76
- for (int b = 0; b < batch; ++b){
72
+ for (int b = 0; b < batch; ++b) {
73
i = width * row + col;
74
res = __int2half_rd(0);
75
k = 0;
76
77
__syncthreads();
78
if (threadIdx.x < blockwidth2)
83
- blockvec[threadIdx.x] = vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 + threadIdx.x];
79
+ blockvec[threadIdx.x] =
80
+ vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 +
81
+ threadIdx.x];
82
__syncthreads();
83
84
while (k < blockwidth2) {
@@ -143,7 +141,8 @@ __global__ void NUQ4MatMulKernel(
141
#ifndef USE_ROCM
142
res = __hadd(__hadd(res2.x, res2.y), res);
143
#else
146
- res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)), res);
144
+ res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)),
145
+ res);
146
#endif
147
148
i += width;
@@ -179,46 +178,38 @@ __global__ void NUQ4MatMulKernel(
178
}
179
}
180
182
-} // namespace squeezellm
183
-} // namespace vllm
181
+} // namespace squeezellm
182
+} // namespace vllm
183
184
// 4-bit matvec kernel (LUT-based)
186
-void squeezellm_gemm(
187
- torch::Tensor vec,
188
- torch::Tensor mat,
189
- torch::Tensor mul,
190
- torch::Tensor lookup_table
191
-) {
185
+void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
186
+ torch::Tensor lookup_table) {
187
int height = mat.size(0);
188
int width = mat.size(1);
189
190
int batch = vec.size(0);
191
int vec_height = vec.size(1);
192
198
- dim3 blocks(
199
- (height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
200
- (width + BLOCKWIDTH - 1) / BLOCKWIDTH
201
- );
193
+ dim3 blocks((height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
194
+ (width + BLOCKWIDTH - 1) / BLOCKWIDTH);
195
dim3 threads(BLOCKWIDTH);
196
197
const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
198
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
199
vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
200
#ifndef USE_ROCM
208
- (half2*) vec.data<at::Half>(),
201
+ (half2*)vec.data<at::Half>(),
202
#else
210
- (__half2*) vec.data_ptr<at::Half>(),
203
+ (__half2*)vec.data_ptr<at::Half>(),
204
#endif
212
- mat.data_ptr<int>(),
205
+ mat.data_ptr<int>(),
206
#ifndef USE_ROCM
214
- (half2*) mul.data<at::Half>(),
215
- (__half*) lookup_table.data<at::Half>(),
207
+ (half2*)mul.data<at::Half>(), (__half*)lookup_table.data<at::Half>(),
208
#else
217
- (float2*) mul.data_ptr<float>(),
218
- (__half*) lookup_table.data_ptr<at::Half>(),
209
+ (float2*)mul.data_ptr<float>(),
210
+ (__half*)lookup_table.data_ptr<at::Half>(),
211
#endif
220
- height, width, batch, vec_height
221
- );
212
+ height, width, batch, vec_height);
213
}
214
215
#undef BLOCKWIDTH
csrc/reduction_utils.cuh
+11
-9
@@ -1,5 +1,6 @@
1
/*
2
- * Adapted from https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
4
* Copyright (c) 2023, The vLLM team.
5
* Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
*
@@ -20,12 +21,12 @@
21
#include "cuda_compat.h"
22
23
namespace vllm {
23
-template<typename T, int numLanes = WARP_SIZE>
24
+template <typename T, int numLanes = WARP_SIZE>
25
__inline__ __device__ T warpReduceSum(T val) {
26
static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
27
"numLanes is not a positive power of 2!");
28
static_assert(numLanes <= WARP_SIZE);
28
- #pragma unroll
29
+#pragma unroll
30
for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
31
val += VLLM_SHFL_XOR_SYNC(val, mask);
32
return val;
@@ -38,22 +39,23 @@ static constexpr int _nextPow2(unsigned int num) {
39
}
40
41
/* Calculate the sum of all elements in a block */
41
-template<typename T, int maxBlockSize = 1024>
42
+template <typename T, int maxBlockSize = 1024>
43
__inline__ __device__ T blockReduceSum(T val) {
44
static_assert(maxBlockSize <= 1024);
45
if constexpr (maxBlockSize > WARP_SIZE) {
46
val = warpReduceSum<T>(val);
46
- // Calculates max number of lanes that need to participate in the last warpReduce
47
+ // Calculates max number of lanes that need to participate in the last
48
+ // warpReduce
49
constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
50
static __shared__ T shared[maxActiveLanes];
51
int lane = threadIdx.x % WARP_SIZE;
52
int wid = threadIdx.x / WARP_SIZE;
51
- if (lane == 0)
52
- shared[wid] = val;
53
+ if (lane == 0) shared[wid] = val;
54
55
__syncthreads();
56
56
- val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane] : (T)(0.0f);
57
+ val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane]
58
+ : (T)(0.0f);
59
val = warpReduceSum<T, _nextPow2(maxActiveLanes)>(val);
60
} else {
61
// A single warpReduce is equal to blockReduce
@@ -62,4 +64,4 @@ __inline__ __device__ T blockReduceSum(T val) {
64
return val;
65
}
66
65
-} // namespace vllm
67
+} // namespace vllm
format.sh
+56
-1
@@ -26,6 +26,7 @@ RUFF_VERSION=$(ruff --version | awk '{print $2}')
26
MYPY_VERSION=$(mypy --version | awk '{print $2}')
27
CODESPELL_VERSION=$(codespell --version)
28
ISORT_VERSION=$(isort --vn)
29
+CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
30
31
# # params: tool name, tool version, required version
32
tool_version_check() {
@@ -40,6 +41,7 @@ tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt |
41
tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
42
tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-dev.txt | cut -d'=' -f3)"
43
tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
44
+tool_version_check "clang-format" "$CLANGFORMAT_VERSION" "$(grep clang-format requirements-dev.txt | cut -d'=' -f3)"
45
46
YAPF_FLAGS=(
47
'--recursive'
@@ -179,7 +181,6 @@ lint_changed() {
181
}
182
183
# Run Ruff
182
-echo 'vLLM ruff:'
184
### This flag lints individual files. --files *must* be the first command line
185
### arg to use this option.
186
if [[ "$1" == '--files' ]]; then
@@ -192,6 +193,7 @@ else
193
# Format only the files that changed in last commit.
194
lint_changed
195
fi
196
+echo 'vLLM ruff: Done'
197
198
# check spelling of specified files
199
isort_check() {
@@ -233,6 +235,59 @@ else
235
fi
236
echo 'vLLM isort: Done'
237
238
+# Clang-format section
239
+# Exclude some files for formatting because they are vendored
240
+# NOTE: Keep up to date with .github/workflows/clang-format.yml
241
+CLANG_FORMAT_EXCLUDES=(
242
+ 'csrc/moe/topk_softmax_kernels.cu'
243
+ 'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
244
+ 'csrc/punica/bgmv/bgmv_config.h'
245
+ 'csrc/punica/bgmv/bgmv_impl.cuh'
246
+ 'csrc/punica/bgmv/vec_dtypes.cuh'
247
+ 'csrc/punica/punica_ops.cu'
248
+ 'csrc/punica/type_convert.h'
249
+)
250
+
251
+# Format specified files with clang-format
252
+clang_format() {
253
+ clang-format -i "$@"
254
+}
255
+
256
+# Format files that differ from main branch with clang-format.
257
+clang_format_changed() {
258
+ # The `if` guard ensures that the list of filenames is not empty, which
259
+ # could cause clang-format to receive 0 positional arguments, making it hang
260
+ # waiting for STDIN.
261
+ #
262
+ # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
263
+ # exist on both branches.
264
+ MERGEBASE="$(git merge-base origin/main HEAD)"
265
+
266
+ # Get the list of changed files, excluding the specified ones
267
+ changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}"))
268
+ if [ -n "$changed_files" ]; then
269
+ echo "$changed_files" | xargs -P 5 clang-format -i
270
+ fi
271
+}
272
+
273
+# Format all files with clang-format
274
+clang_format_all() {
275
+ find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
276
+ | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") \
277
+ | xargs clang-format -i
278
+}
279
+
280
+# Run clang-format
281
+if [[ "$1" == '--files' ]]; then
282
+ clang_format "${@:2}"
283
+elif [[ "$1" == '--all' ]]; then
284
+ clang_format_all
285
+else
286
+ clang_format_changed
287
+fi
288
+echo 'vLLM clang-format: Done'
289
+
290
+
291
if ! git diff --quiet &>/dev/null; then
292
echo 'Reformatted files. Please review and stage the changes.'
293
echo 'Changes not staged for commit:'
requirements-dev.txt
+1
@@ -5,6 +5,7 @@ tomli==2.0.1
5
ruff==0.1.5
6
codespell==2.2.6
7
isort==5.13.2
8
+clang-format==18.1.5
9
10
# type checking
11
mypy==1.9.0