@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ffffccb7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Kernel] Implement fallback for FP8 channelwise using torch._scaled_mm (#6552)

Tyler Michael Smith committed Jul 18, 2024 at 19:52 UTC 4ffffccb7eca8a8911ab37b7e15e5d0210838ceb
2 files changed +40 -21
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
-11
@@ -23,16 +23,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
23 self.is_static_input_scheme = is_static_input_scheme
24 self.cutlass_fp8_supported = cutlass_fp8_supported()
25
26 - # On Lovelace, fail for now if channelwise.
27 - # TODO: (@tms) fallback
28 - if (not self.cutlass_fp8_supported
29 - and self.strategy == QuantizationStrategy.CHANNEL):
30 - raise ValueError(
31 - "Channelwise fp8 quantization requires vLLM's custom "
32 - "cutlass kernels, which are not supported on your device."
33 - "Consider quantizing with per tensor scales or upgrading "
34 - "to Hopper.")
35 -
26 def get_min_capability(self) -> int:
27 # lovelace and up
28 return 89
@@ -53,7 +43,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
43
44 # If channelwise, scales are already lined up, so just transpose.
45 elif self.strategy == QuantizationStrategy.CHANNEL:
56 - assert self.cutlass_fp8_supported
46 weight = layer.weight
47 layer.weight = Parameter(weight.t(), requires_grad=False)
48
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+40 -10
@@ -124,20 +124,50 @@ def apply_fp8_linear(
124 bias=bias)
125
126 else:
127 + # Note: we pad the input because torch._scaled_mm is more performant
128 + # for matrices with batch dimension > 16.
129 + # This could change in the future.
130 qinput, x_scale = ops.scaled_fp8_quant(input,
131 input_scale,
132 batch_dim_padding=17)
133
131 - # Fused GEMM_DQ -- note we padded the input above because
132 - # torch._scaled_mm is more performant for matrices with
133 - # batch dimension > 16. Note that this could change
134 - # in the future.
135 - output, _ = torch._scaled_mm(qinput,
136 - weight,
137 - out_dtype=input.dtype,
138 - scale_a=x_scale,
139 - scale_b=weight_scale,
140 - bias=bias)
134 + if weight_scale.numel() == 1:
135 + # Fused GEMM_DQ
136 + output, _ = torch._scaled_mm(qinput,
137 + weight,
138 + out_dtype=input.dtype,
139 + scale_a=x_scale,
140 + scale_b=weight_scale,
141 + bias=bias)
142 + else:
143 + # Fallback for channelwise case, where the weight scales are
144 + # applied separately.
145 +
146 + # Symmetric quantized GEMM by definition computes the following:
147 + # C = (s_x * X) (s_w * W) + bias
148 + # This is equivalent to dequantizing the weights and activations
149 + # before applying a GEMM.
150 + #
151 + # In order to compute quantized operands, a quantized kernel
152 + # will rewrite the above like so:
153 + # C = s_w * s_x * (X * W) + bias
154 + #
155 + # For the scaled_mm fallback case, we break this down, since it
156 + # does not support s_w being a vector.
157 +
158 + # This computes C = sx * (X * W).
159 + # Output in fp32 to allow subsequent ops to happen in-place
160 + output, _ = torch._scaled_mm(qinput,
161 + weight,
162 + out_dtype=torch.float32,
163 + scale_a=x_scale)
164 +
165 + # C = sw * sx * (X * W)
166 + output = output * weight_scale.t()
167 + if bias is not None:
168 + # C = sw * sx * (X * W) + bias
169 + output = output + bias
170 + output = output.to(dtype=input.dtype)
171
172 return torch.narrow(output, 0, 0, input.shape[0])
173