A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix][Kernel] Fix per-token/per-channel quantization for Hopper scaled mm (#12696)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Feb 3, 2025 at 16:04 UTC
c11de33dad217bca79225128059b6fac7e1b2519
1 file changed
+24
-35
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+24
-35
@@ -16,29 +16,11 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
16
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
17
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
18
19
- using GroupShape = std::array<int64_t, 2>;
20
-
19
int M = a.size(0), N = b.size(1), K = a.size(1);
20
23
- GroupShape a_scale_group_shape = [&, &s = a_scales]() -> GroupShape {
24
- if (s.numel() == 1) return {M, K}; // tensor-wise
25
- if (s.dim() == 2)
26
- return {ceil_div(a.size(0), s.size(0)), ceil_div(a.size(1), s.size(1))};
27
- TORCH_CHECK(false, "Unsupported scale shape for scale_a");
28
- }();
29
-
30
- GroupShape b_scale_group_shape = [&, &s = b_scales]() -> GroupShape {
31
- if (s.numel() == 1) return {K, N}; // tensor-wise
32
- if (s.dim() == 2)
33
- return {ceil_div(b.size(0), s.size(0)), ceil_div(b.size(1), s.size(1))};
34
- TORCH_CHECK(false, "Unsupported scale shape for scale_b");
35
- }();
36
-
37
- if ((a_scale_group_shape == GroupShape{M, K} ||
38
- a_scale_group_shape == GroupShape{1, K}) &&
39
- (b_scale_group_shape == GroupShape{K, N} ||
40
- b_scale_group_shape == GroupShape{K, 1})) {
41
- // "standard per-tensor/per-token/per-channel" scaling
21
+ if ((a_scales.numel() == 1 || a_scales.numel() == a.size(0)) &&
22
+ (b_scales.numel() == 1 || b_scales.numel() == b.size(1))) {
23
+ // Standard per-tensor/per-token/per-channel scaling
24
TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
25
if (a.dtype() == torch::kFloat8_e4m3fn) {
26
vllm::cutlass_scaled_mm_sm90_fp8(c, a, b, a_scales, b_scales, bias);
@@ -46,25 +28,32 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
28
TORCH_CHECK(a.dtype() == torch::kInt8);
29
vllm::cutlass_scaled_mm_sm90_int8(c, a, b, a_scales, b_scales, bias);
30
}
49
- } else if (a_scale_group_shape == GroupShape{1, 128} &&
50
- b_scale_group_shape == GroupShape{128, 128}) {
31
+ } else {
32
+ using GroupShape = std::array<int64_t, 2>;
33
+ auto make_group_shape = [](torch::Tensor const& x,
34
+ torch::Tensor const& s) -> GroupShape {
35
+ TORCH_CHECK(s.dim() == 2, "cutlass_scaled_mm group scales must be 2D");
36
+ return {ceil_div(x.size(0), s.size(0)), ceil_div(x.size(1), s.size(1))};
37
+ };
38
+
39
+ GroupShape a_scale_group_shape = make_group_shape(a, a_scales);
40
+ GroupShape b_scale_group_shape = make_group_shape(b, b_scales);
41
+
42
// 1x128 per-token group scales for activations
43
// 128x128 blockwise scales for weights
53
- TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn &&
54
- b.dtype() == torch::kFloat8_e4m3fn,
55
- "Currently only FP8 is supported for A group shape 1x128 and "
56
- "B group shape 128x128");
57
- TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
58
-
59
- vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60
- } else {
61
- TORCH_CHECK(false,
62
- "Unsupported scale group shapes for CUTLASS 3.x GEMM.\n "
63
- "a_scale_group_shape must be [1, 128], got: [",
44
+ TORCH_CHECK((a_scale_group_shape == GroupShape{1, 128} &&
45
+ b_scale_group_shape == GroupShape{128, 128} &&
46
+ a.dtype() == torch::kFloat8_e4m3fn &&
47
+ b.dtype() == torch::kFloat8_e4m3fn),
48
+ "cutlass_scaled_mm only supports datatype float8_e4m3fn.\n"
49
+ "a_scale_group_shape must be [1, 128]. Got: [",
50
a_scale_group_shape[0], ", ", a_scale_group_shape[1],
51
"]\n"
66
- "b_scale_group_shape must be [128, 128], got: [",
52
+ "b_scale_group_shape must be [128, 128]. Got: [",
53
b_scale_group_shape[0], ", ", b_scale_group_shape[1], "]");
54
+ TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
55
+
56
+ vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
57
}
58
}
59