@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c11de33dad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix][Kernel] Fix per-token/per-channel quantization for Hopper scaled mm (#12696)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Feb 3, 2025 at 16:04 UTC c11de33dad217bca79225128059b6fac7e1b2519
1 file changed +24 -35
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+24 -35
@@ -16,29 +16,11 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
16 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
17 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
18
19 - using GroupShape = std::array<int64_t, 2>;
20 -
19 int M = a.size(0), N = b.size(1), K = a.size(1);
20
23 - GroupShape a_scale_group_shape = [&, &s = a_scales]() -> GroupShape {
24 - if (s.numel() == 1) return {M, K}; // tensor-wise
25 - if (s.dim() == 2)
26 - return {ceil_div(a.size(0), s.size(0)), ceil_div(a.size(1), s.size(1))};
27 - TORCH_CHECK(false, "Unsupported scale shape for scale_a");
28 - }();
29 -
30 - GroupShape b_scale_group_shape = [&, &s = b_scales]() -> GroupShape {
31 - if (s.numel() == 1) return {K, N}; // tensor-wise
32 - if (s.dim() == 2)
33 - return {ceil_div(b.size(0), s.size(0)), ceil_div(b.size(1), s.size(1))};
34 - TORCH_CHECK(false, "Unsupported scale shape for scale_b");
35 - }();
36 -
37 - if ((a_scale_group_shape == GroupShape{M, K} ||
38 - a_scale_group_shape == GroupShape{1, K}) &&
39 - (b_scale_group_shape == GroupShape{K, N} ||
40 - b_scale_group_shape == GroupShape{K, 1})) {
41 - // "standard per-tensor/per-token/per-channel" scaling
21 + if ((a_scales.numel() == 1 || a_scales.numel() == a.size(0)) &&
22 + (b_scales.numel() == 1 || b_scales.numel() == b.size(1))) {
23 + // Standard per-tensor/per-token/per-channel scaling
24 TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
25 if (a.dtype() == torch::kFloat8_e4m3fn) {
26 vllm::cutlass_scaled_mm_sm90_fp8(c, a, b, a_scales, b_scales, bias);
@@ -46,25 +28,32 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
28 TORCH_CHECK(a.dtype() == torch::kInt8);
29 vllm::cutlass_scaled_mm_sm90_int8(c, a, b, a_scales, b_scales, bias);
30 }
49 - } else if (a_scale_group_shape == GroupShape{1, 128} &&
50 - b_scale_group_shape == GroupShape{128, 128}) {
31 + } else {
32 + using GroupShape = std::array<int64_t, 2>;
33 + auto make_group_shape = [](torch::Tensor const& x,
34 + torch::Tensor const& s) -> GroupShape {
35 + TORCH_CHECK(s.dim() == 2, "cutlass_scaled_mm group scales must be 2D");
36 + return {ceil_div(x.size(0), s.size(0)), ceil_div(x.size(1), s.size(1))};
37 + };
38 +
39 + GroupShape a_scale_group_shape = make_group_shape(a, a_scales);
40 + GroupShape b_scale_group_shape = make_group_shape(b, b_scales);
41 +
42 // 1x128 per-token group scales for activations
43 // 128x128 blockwise scales for weights
53 - TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn &&
54 - b.dtype() == torch::kFloat8_e4m3fn,
55 - "Currently only FP8 is supported for A group shape 1x128 and "
56 - "B group shape 128x128");
57 - TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
58 -
59 - vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
60 - } else {
61 - TORCH_CHECK(false,
62 - "Unsupported scale group shapes for CUTLASS 3.x GEMM.\n "
63 - "a_scale_group_shape must be [1, 128], got: [",
44 + TORCH_CHECK((a_scale_group_shape == GroupShape{1, 128} &&
45 + b_scale_group_shape == GroupShape{128, 128} &&
46 + a.dtype() == torch::kFloat8_e4m3fn &&
47 + b.dtype() == torch::kFloat8_e4m3fn),
48 + "cutlass_scaled_mm only supports datatype float8_e4m3fn.\n"
49 + "a_scale_group_shape must be [1, 128]. Got: [",
50 a_scale_group_shape[0], ", ", a_scale_group_shape[1],
51 "]\n"
66 - "b_scale_group_shape must be [128, 128], got: [",
52 + "b_scale_group_shape must be [128, 128]. Got: [",
53 b_scale_group_shape[0], ", ", b_scale_group_shape[1], "]");
54 + TORCH_CHECK(!bias, "Bias not yet supported blockwise scaled_mm");
55 +
56 + vllm::cutlass_scaled_mm_blockwise_sm90_fp8(c, a, b, a_scales, b_scales);
57 }
58 }
59