@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb5741ad42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3 (#12587)

Integrates the block-quantized kernels introduced in https://github.com/vllm-project/vllm/pull/11868 for use in linear layers. Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Jan 31, 2025 at 18:29 UTC eb5741ad422f04d0bac60c9b6c07183e0431ce8c
8 files changed +160 -37
csrc/ops.h
+1
@@ -153,6 +153,7 @@ torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
153
154 #ifndef USE_ROCM
155 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
156 +bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability);
157
158 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
159 torch::Tensor const& b, torch::Tensor const& a_scales,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+7 -1
@@ -58,7 +58,13 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
58
59 vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60 } else {
61 - TORCH_CHECK(false, "Unsupported scale group shapes for CUTLASS 3.x GEMM");
61 + TORCH_CHECK(false,
62 + "Unsupported scale group shapes for CUTLASS 3.x GEMM.\n "
63 + "a_scale_group_shape must be [1, 128], got: [",
64 + a_scale_group_shape[0], ", ", a_scale_group_shape[1],
65 + "]\n"
66 + "b_scale_group_shape must be [128, 128], got: [",
67 + b_scale_group_shape[0], ", ", b_scale_group_shape[1], "]");
68 }
69 }
70
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+14 -1
@@ -81,6 +81,19 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
81 return false;
82 }
83
84 +bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability) {
85 + // CUTLASS block-quantized FP8 kernels need at least CUDA 12.0
86 + // and at least SM90 (Hopper)
87 +
88 +#if defined CUDA_VERSION
89 + if (cuda_device_capability >= 90) {
90 + return CUDA_VERSION >= 12000;
91 + }
92 +#endif
93 +
94 + return false;
95 +}
96 +
97 void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
98 torch::Tensor const& b, torch::Tensor const& a_scales,
99 torch::Tensor const& b_scales,
@@ -212,4 +225,4 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
225 "No compiled cutlass_scaled_mm_azp for a compute capability less than "
226 "CUDA device capability: ",
227 version_num);
215 -}
\ No newline at end of file
228 +}
csrc/torch_bindings.cpp
+7
@@ -324,6 +324,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
324 ops.def("cutlass_scaled_mm_supports_fp8(int cuda_device_capability) -> bool");
325 ops.impl("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
326
327 + // Check if cutlass scaled_mm supports block quantization (used by DeepSeekV3)
328 + ops.def(
329 + "cutlass_scaled_mm_supports_block_fp8(int cuda_device_capability) -> "
330 + "bool");
331 + ops.impl("cutlass_scaled_mm_supports_block_fp8",
332 + &cutlass_scaled_mm_supports_fp8);
333 +
334 // Check if cutlass sparse scaled_mm is supported for CUDA devices of the
335 // given capability
336 ops.def(
vllm/_custom_ops.py
+5
@@ -435,6 +435,11 @@ def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
435 return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
436
437
438 +def cutlass_scaled_mm_supports_block_fp8(cuda_device_capability: int) -> bool:
439 + return torch.ops._C.cutlass_scaled_mm_supports_block_fp8(
440 + cuda_device_capability)
441 +
442 +
443 def cutlass_scaled_mm(a: torch.Tensor,
444 b: torch.Tensor,
445 scale_a: torch.Tensor,
vllm/model_executor/layers/quantization/fp8.py
+4 -1
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
21 is_layer_skipped)
22 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
23 all_close_1d, apply_fp8_linear, convert_to_channelwise,
24 - cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
24 + cutlass_block_fp8_supported, cutlass_fp8_supported,
25 + normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
26 requantize_with_max_scale)
27 from vllm.model_executor.parameter import (BlockQuantScaleParameter,
28 ModelWeightParameter,
@@ -133,6 +134,7 @@ class Fp8LinearMethod(LinearMethodBase):
134 def __init__(self, quant_config: Fp8Config):
135 self.quant_config = quant_config
136 self.cutlass_fp8_supported = cutlass_fp8_supported()
137 + self.cutlass_block_fp8_supported = cutlass_block_fp8_supported()
138
139 # For GPUs that lack FP8 hardware support, we can leverage the Marlin
140 # kernel for fast weight-only FP8 quantization
@@ -359,6 +361,7 @@ class Fp8LinearMethod(LinearMethodBase):
361 weight_scale=layer.weight_scale_inv,
362 input_scale=layer.input_scale,
363 bias=bias,
364 + cutlass_block_fp8_supported=self.cutlass_block_fp8_supported,
365 )
366
367 return apply_fp8_linear(
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+112 -34
@@ -8,6 +8,7 @@ import torch
8 import triton
9 import triton.language as tl
10
11 +from vllm import _custom_ops as ops
12 from vllm.logger import init_logger
13 from vllm.platforms import current_platform
14
@@ -21,20 +22,34 @@ def apply_w8a8_block_fp8_linear(
22 weight_scale: torch.Tensor,
23 input_scale: Optional[torch.Tensor] = None,
24 bias: Optional[torch.Tensor] = None,
25 + cutlass_block_fp8_supported: bool = True,
26 ) -> torch.Tensor:
27 assert input_scale is None
28 # View input as 2D matrix for fp8 methods
29 input_2d = input.view(-1, input.shape[-1])
30 output_shape = [*input.shape[:-1], weight.shape[0]]
31
30 - q_input, x_scale = per_token_group_quant_fp8(input_2d, block_size[1])
31 - output = w8a8_block_fp8_matmul(q_input,
32 - weight,
33 - x_scale,
34 - weight_scale,
35 - block_size,
36 - output_dtype=input.dtype)
37 -
32 + shape_supported_by_cutlass = (weight.shape[0] % 128 == 0
33 + and weight.shape[1] % 128 == 0)
34 + if cutlass_block_fp8_supported and shape_supported_by_cutlass:
35 + q_input, x_scale = per_token_group_quant_fp8(input_2d,
36 + block_size[1],
37 + column_major_scales=True)
38 + output = ops.cutlass_scaled_mm(q_input,
39 + weight.T,
40 + out_dtype=input.dtype,
41 + scale_a=x_scale,
42 + scale_b=weight_scale.T)
43 + else:
44 + q_input, x_scale = per_token_group_quant_fp8(input_2d,
45 + block_size[1],
46 + column_major_scales=False)
47 + output = w8a8_block_fp8_matmul(q_input,
48 + weight,
49 + x_scale,
50 + weight_scale,
51 + block_size,
52 + output_dtype=input.dtype)
53 if bias is not None:
54 output = output + bias
55 return output.to(dtype=input.dtype).view(*output_shape)
@@ -98,10 +113,7 @@ def _per_token_group_quant_fp8(
113 y_ptr,
114 y_q_ptr,
115 y_s_ptr,
101 - # Stride of input
102 - y_stride,
103 - # Columns of input
104 - N,
116 + group_size,
117 # Avoid to divide zero
118 eps,
119 # Information for float8
@@ -116,12 +128,60 @@ def _per_token_group_quant_fp8(
128 """
129 # Map the program id to the row of X and Y it should compute.
130 g_id = tl.program_id(0)
119 - y_ptr += g_id * y_stride
120 - y_q_ptr += g_id * y_stride
131 + y_ptr += g_id * group_size
132 + y_q_ptr += g_id * group_size
133 y_s_ptr += g_id
134
135 cols = tl.arange(0, BLOCK) # N <= BLOCK
124 - mask = cols < N
136 + mask = cols < group_size
137 +
138 + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
139 + # Quant
140 + _absmax = tl.maximum(tl.max(tl.abs(y)), eps)
141 + y_s = _absmax / fp8_max
142 + y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
143 +
144 + tl.store(y_q_ptr + cols, y_q, mask=mask)
145 + tl.store(y_s_ptr, y_s)
146 +
147 +
148 +@triton.jit
149 +def _per_token_group_quant_fp8_colmajor(
150 + # Pointers to inputs and output
151 + y_ptr,
152 + y_q_ptr,
153 + y_s_ptr,
154 + group_size,
155 + # Num columns of y
156 + y_num_columns,
157 + # Stride from one column to the next of y_s
158 + y_s_col_stride,
159 + # Avoid to divide zero
160 + eps,
161 + # Information for float8
162 + fp8_min,
163 + fp8_max,
164 + # Meta-parameters
165 + BLOCK: tl.constexpr,
166 +):
167 + """A Triton-accelerated function to perform per-token-group
168 + quantization on a tensor.
169 + This function converts the tensor values into float8 values.
170 + """
171 + # Map the program id to the row of X and Y it should compute.
172 + g_id = tl.program_id(0)
173 + y_ptr += g_id * group_size
174 + y_q_ptr += g_id * group_size
175 +
176 + # Convert g_id the flattened block coordinate to 2D so we can index
177 + # into the output y_scales matrix
178 + blocks_per_row = y_num_columns // group_size
179 + scale_col = g_id % blocks_per_row
180 + scale_row = g_id // blocks_per_row
181 + y_s_ptr += scale_col * y_s_col_stride + scale_row
182 +
183 + cols = tl.arange(0, BLOCK) # group_size <= BLOCK
184 + mask = cols < group_size
185
186 y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
187 # Quant
@@ -138,12 +198,13 @@ def per_token_group_quant_fp8(
198 group_size: int,
199 eps: float = 1e-10,
200 dtype: Optional[torch.dtype] = None,
201 + column_major_scales: bool = False,
202 ) -> Tuple[torch.Tensor, torch.Tensor]:
203 """Function to perform per-token-group quantization on an input tensor `x`.
204 It converts the tensor values into signed float8 values and returns the
205 quantized tensor along with the scaling factor used for quantization.
206 Args:
146 - x: The input tenosr with ndim >= 2.
207 + x: The input tensor with ndim >= 2.
208 group_size: The group size used for quantization.
209 eps: The minimum to avoid dividing zero.
210 dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn`
@@ -167,29 +228,46 @@ def per_token_group_quant_fp8(
228 x_q = torch.empty_like(x, device=x.device, dtype=dtype)
229 M = x.numel() // group_size
230 N = group_size
170 - x_s = torch.empty(
171 - x.shape[:-1] + (x.shape[-1] // group_size, ),
172 - device=x.device,
173 - dtype=torch.float32,
174 - )
231 + if column_major_scales:
232 + shape = (x.shape[-1] // group_size, ) + x.shape[:-1]
233 + x_s = torch.empty(shape, device=x.device,
234 + dtype=torch.float32).permute(-1, -2)
235 + else:
236 + shape = x.shape[:-1] + (x.shape[-1] // group_size, )
237 + x_s = torch.empty(shape, device=x.device, dtype=torch.float32)
238
239 BLOCK = triton.next_power_of_2(N)
240 # heuristics for number of warps
241 num_warps = min(max(BLOCK // 256, 1), 8)
242 num_stages = 1
180 - _per_token_group_quant_fp8[(M, )](
181 - x,
182 - x_q,
183 - x_s,
184 - group_size,
185 - N,
186 - eps,
187 - fp8_min=fp8_min,
188 - fp8_max=fp8_max,
189 - BLOCK=BLOCK,
190 - num_warps=num_warps,
191 - num_stages=num_stages,
192 - )
243 + if column_major_scales:
244 + _per_token_group_quant_fp8_colmajor[(M, )](
245 + x,
246 + x_q,
247 + x_s,
248 + group_size,
249 + x.shape[1],
250 + x_s.stride(1),
251 + eps,
252 + fp8_min=fp8_min,
253 + fp8_max=fp8_max,
254 + BLOCK=BLOCK,
255 + num_warps=num_warps,
256 + num_stages=num_stages,
257 + )
258 + else:
259 + _per_token_group_quant_fp8[(M, )](
260 + x,
261 + x_q,
262 + x_s,
263 + group_size,
264 + eps,
265 + fp8_min=fp8_min,
266 + fp8_max=fp8_max,
267 + BLOCK=BLOCK,
268 + num_warps=num_warps,
269 + num_stages=num_stages,
270 + )
271
272 return x_q, x_s
273
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+10
@@ -30,6 +30,16 @@ def cutlass_fp8_supported() -> bool:
30 return ops.cutlass_scaled_mm_supports_fp8(capability)
31
32
33 +def cutlass_block_fp8_supported() -> bool:
34 + if not current_platform.is_cuda():
35 + return False
36 +
37 + capability_tuple = current_platform.get_device_capability()
38 + capability = -1 if capability_tuple is None else capability_tuple.to_int()
39 +
40 + return ops.cutlass_scaled_mm_supports_block_fp8(capability)
41 +
42 +
43 def per_tensor_dequantize(
44 tensor: torch.Tensor, inv_scale: Union[float,
45 torch.Tensor]) -> torch.Tensor: