A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3 (#12587)
Integrates the block-quantized kernels introduced in https://github.com/vllm-project/vllm/pull/11868 for use in linear layers. Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Jan 31, 2025 at 18:29 UTC
eb5741ad422f04d0bac60c9b6c07183e0431ce8c
8 files changed
+160
-37
csrc/ops.h
+1
@@ -153,6 +153,7 @@ torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
153
154
#ifndef USE_ROCM
155
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
156
+bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability);
157
158
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
159
torch::Tensor const& b, torch::Tensor const& a_scales,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+7
-1
@@ -58,7 +58,13 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
58
59
vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60
} else {
61
- TORCH_CHECK(false, "Unsupported scale group shapes for CUTLASS 3.x GEMM");
61
+ TORCH_CHECK(false,
62
+ "Unsupported scale group shapes for CUTLASS 3.x GEMM.\n "
63
+ "a_scale_group_shape must be [1, 128], got: [",
64
+ a_scale_group_shape[0], ", ", a_scale_group_shape[1],
65
+ "]\n"
66
+ "b_scale_group_shape must be [128, 128], got: [",
67
+ b_scale_group_shape[0], ", ", b_scale_group_shape[1], "]");
68
}
69
}
70
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+14
-1
@@ -81,6 +81,19 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
81
return false;
82
}
83
84
+bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability) {
85
+ // CUTLASS block-quantized FP8 kernels need at least CUDA 12.0
86
+ // and at least SM90 (Hopper)
87
+
88
+#if defined CUDA_VERSION
89
+ if (cuda_device_capability >= 90) {
90
+ return CUDA_VERSION >= 12000;
91
+ }
92
+#endif
93
+
94
+ return false;
95
+}
96
+
97
void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
98
torch::Tensor const& b, torch::Tensor const& a_scales,
99
torch::Tensor const& b_scales,
@@ -212,4 +225,4 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
225
"No compiled cutlass_scaled_mm_azp for a compute capability less than "
226
"CUDA device capability: ",
227
version_num);
215
-}
\ No newline at end of file
228
+}
csrc/torch_bindings.cpp
+7
@@ -324,6 +324,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
324
ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
325
ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
326
327
+ // Check if cutlass scaled_mm supports block quantization (used by DeepSeekV3)
328
+ ops.def(
329
+ "cutlass_scaled_mm_supports_block_fp8(int cuda_device_capability) -> "
330
+ "bool");
331
+ ops.impl("cutlass_scaled_mm_supports_block_fp8",
332
+ &cutlass_scaled_mm_supports_fp8);
333
+
334
// Check if cutlass sparse scaled_mm is supported for CUDA devices of the
335
// given capability
336
ops.def(
vllm/_custom_ops.py
+5
@@ -435,6 +435,11 @@ def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
435
return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
436
437
438
+def cutlass_scaled_mm_supports_block_fp8(cuda_device_capability: int) -> bool:
439
+ return torch.ops._C.cutlass_scaled_mm_supports_block_fp8(
440
+ cuda_device_capability)
441
+
442
+
443
def cutlass_scaled_mm(a: torch.Tensor,
444
b: torch.Tensor,
445
scale_a: torch.Tensor,
vllm/model_executor/layers/quantization/fp8.py
+4
-1
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
21
is_layer_skipped)
22
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
23
all_close_1d, apply_fp8_linear, convert_to_channelwise,
24
- cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
24
+ cutlass_block_fp8_supported, cutlass_fp8_supported,
25
+ normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
26
requantize_with_max_scale)
27
from vllm.model_executor.parameter import (BlockQuantScaleParameter,
28
ModelWeightParameter,
@@ -133,6 +134,7 @@ class Fp8LinearMethod(LinearMethodBase):
134
def __init__(self, quant_config: Fp8Config):
135
self.quant_config = quant_config
136
self.cutlass_fp8_supported = cutlass_fp8_supported()
137
+ self.cutlass_block_fp8_supported = cutlass_block_fp8_supported()
138
139
# For GPUs that lack FP8 hardware support, we can leverage the Marlin
140
# kernel for fast weight-only FP8 quantization
@@ -359,6 +361,7 @@ class Fp8LinearMethod(LinearMethodBase):
361
weight_scale=layer.weight_scale_inv,
362
input_scale=layer.input_scale,
363
bias=bias,
364
+ cutlass_block_fp8_supported=self.cutlass_block_fp8_supported,
365
)
366
367
return apply_fp8_linear(
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+112
-34
@@ -8,6 +8,7 @@ import torch
8
import triton
9
import triton.language as tl
10
11
+from vllm import _custom_ops as ops
12
from vllm.logger import init_logger
13
from vllm.platforms import current_platform
14
@@ -21,20 +22,34 @@ def apply_w8a8_block_fp8_linear(
22
weight_scale: torch.Tensor,
23
input_scale: Optional[torch.Tensor] = None,
24
bias: Optional[torch.Tensor] = None,
25
+ cutlass_block_fp8_supported: bool = True,
26
) -> torch.Tensor:
27
assert input_scale is None
28
# View input as 2D matrix for fp8 methods
29
input_2d = input.view(-1, input.shape[-1])
30
output_shape = [*input.shape[:-1], weight.shape[0]]
31
30
- q_input, x_scale = per_token_group_quant_fp8(input_2d, block_size[1])
31
- output = w8a8_block_fp8_matmul(q_input,
32
- weight,
33
- x_scale,
34
- weight_scale,
35
- block_size,
36
- output_dtype=input.dtype)
37
-
32
+ shape_supported_by_cutlass = (weight.shape[0] % 128 == 0
33
+ and weight.shape[1] % 128 == 0)
34
+ if cutlass_block_fp8_supported and shape_supported_by_cutlass:
35
+ q_input, x_scale = per_token_group_quant_fp8(input_2d,
36
+ block_size[1],
37
+ column_major_scales=True)
38
+ output = ops.cutlass_scaled_mm(q_input,
39
+ weight.T,
40
+ out_dtype=input.dtype,
41
+ scale_a=x_scale,
42
+ scale_b=weight_scale.T)
43
+ else:
44
+ q_input, x_scale = per_token_group_quant_fp8(input_2d,
45
+ block_size[1],
46
+ column_major_scales=False)
47
+ output = w8a8_block_fp8_matmul(q_input,
48
+ weight,
49
+ x_scale,
50
+ weight_scale,
51
+ block_size,
52
+ output_dtype=input.dtype)
53
if bias is not None:
54
output = output + bias
55
return output.to(dtype=input.dtype).view(*output_shape)
@@ -98,10 +113,7 @@ def _per_token_group_quant_fp8(
113
y_ptr,
114
y_q_ptr,
115
y_s_ptr,
101
- # Stride of input
102
- y_stride,
103
- # Columns of input
104
- N,
116
+ group_size,
117
# Avoid to divide zero
118
eps,
119
# Information for float8
@@ -116,12 +128,60 @@ def _per_token_group_quant_fp8(
128
"""
129
# Map the program id to the row of X and Y it should compute.
130
g_id = tl.program_id(0)
119
- y_ptr += g_id * y_stride
120
- y_q_ptr += g_id * y_stride
131
+ y_ptr += g_id * group_size
132
+ y_q_ptr += g_id * group_size
133
y_s_ptr += g_id
134
135
cols = tl.arange(0, BLOCK) # N <= BLOCK
124
- mask = cols < N
136
+ mask = cols < group_size
137
+
138
+ y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
139
+ # Quant
140
+ _absmax = tl.maximum(tl.max(tl.abs(y)), eps)
141
+ y_s = _absmax / fp8_max
142
+ y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
143
+
144
+ tl.store(y_q_ptr + cols, y_q, mask=mask)
145
+ tl.store(y_s_ptr, y_s)
146
+
147
+
148
+@triton.jit
149
+def _per_token_group_quant_fp8_colmajor(
150
+ # Pointers to inputs and output
151
+ y_ptr,
152
+ y_q_ptr,
153
+ y_s_ptr,
154
+ group_size,
155
+ # Num columns of y
156
+ y_num_columns,
157
+ # Stride from one column to the next of y_s
158
+ y_s_col_stride,
159
+ # Avoid to divide zero
160
+ eps,
161
+ # Information for float8
162
+ fp8_min,
163
+ fp8_max,
164
+ # Meta-parameters
165
+ BLOCK: tl.constexpr,
166
+):
167
+ """A Triton-accelerated function to perform per-token-group
168
+ quantization on a tensor.
169
+ This function converts the tensor values into float8 values.
170
+ """
171
+ # Map the program id to the row of X and Y it should compute.
172
+ g_id = tl.program_id(0)
173
+ y_ptr += g_id * group_size
174
+ y_q_ptr += g_id * group_size
175
+
176
+ # Convert g_id the flattened block coordinate to 2D so we can index
177
+ # into the output y_scales matrix
178
+ blocks_per_row = y_num_columns // group_size
179
+ scale_col = g_id % blocks_per_row
180
+ scale_row = g_id // blocks_per_row
181
+ y_s_ptr += scale_col * y_s_col_stride + scale_row
182
+
183
+ cols = tl.arange(0, BLOCK) # group_size <= BLOCK
184
+ mask = cols < group_size
185
186
y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
187
# Quant
@@ -138,12 +198,13 @@ def per_token_group_quant_fp8(
198
group_size: int,
199
eps: float = 1e-10,
200
dtype: Optional[torch.dtype] = None,
201
+ column_major_scales: bool = False,
202
) -> Tuple[torch.Tensor, torch.Tensor]:
203
"""Function to perform per-token-group quantization on an input tensor `x`.
204
It converts the tensor values into signed float8 values and returns the
205
quantized tensor along with the scaling factor used for quantization.
206
Args:
146
- x: The input tenosr with ndim >= 2.
207
+ x: The input tensor with ndim >= 2.
208
group_size: The group size used for quantization.
209
eps: The minimum to avoid dividing zero.
210
dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn`
@@ -167,29 +228,46 @@ def per_token_group_quant_fp8(
228
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
229
M = x.numel() // group_size
230
N = group_size
170
- x_s = torch.empty(
171
- x.shape[:-1] + (x.shape[-1] // group_size, ),
172
- device=x.device,
173
- dtype=torch.float32,
174
- )
231
+ if column_major_scales:
232
+ shape = (x.shape[-1] // group_size, ) + x.shape[:-1]
233
+ x_s = torch.empty(shape, device=x.device,
234
+ dtype=torch.float32).permute(-1, -2)
235
+ else:
236
+ shape = x.shape[:-1] + (x.shape[-1] // group_size, )
237
+ x_s = torch.empty(shape, device=x.device, dtype=torch.float32)
238
239
BLOCK = triton.next_power_of_2(N)
240
# heuristics for number of warps
241
num_warps = min(max(BLOCK // 256, 1), 8)
242
num_stages = 1
180
- _per_token_group_quant_fp8[(M, )](
181
- x,
182
- x_q,
183
- x_s,
184
- group_size,
185
- N,
186
- eps,
187
- fp8_min=fp8_min,
188
- fp8_max=fp8_max,
189
- BLOCK=BLOCK,
190
- num_warps=num_warps,
191
- num_stages=num_stages,
192
- )
243
+ if column_major_scales:
244
+ _per_token_group_quant_fp8_colmajor[(M, )](
245
+ x,
246
+ x_q,
247
+ x_s,
248
+ group_size,
249
+ x.shape[1],
250
+ x_s.stride(1),
251
+ eps,
252
+ fp8_min=fp8_min,
253
+ fp8_max=fp8_max,
254
+ BLOCK=BLOCK,
255
+ num_warps=num_warps,
256
+ num_stages=num_stages,
257
+ )
258
+ else:
259
+ _per_token_group_quant_fp8[(M, )](
260
+ x,
261
+ x_q,
262
+ x_s,
263
+ group_size,
264
+ eps,
265
+ fp8_min=fp8_min,
266
+ fp8_max=fp8_max,
267
+ BLOCK=BLOCK,
268
+ num_warps=num_warps,
269
+ num_stages=num_stages,
270
+ )
271
272
return x_q, x_s
273
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+10
@@ -30,6 +30,16 @@ def cutlass_fp8_supported() -> bool:
30
return ops.cutlass_scaled_mm_supports_fp8(capability)
31
32
33
+def cutlass_block_fp8_supported() -> bool:
34
+ if not current_platform.is_cuda():
35
+ return False
36
+
37
+ capability_tuple = current_platform.get_device_capability()
38
+ capability = -1 if capability_tuple is None else capability_tuple.to_int()
39
+
40
+ return ops.cutlass_scaled_mm_supports_block_fp8(capability)
41
+
42
+
43
def per_tensor_dequantize(
44
tensor: torch.Tensor, inv_scale: Union[float,
45
torch.Tensor]) -> torch.Tensor: