A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Kernel] Implement fallback for FP8 channelwise using torch._scaled_mm (#6552)
Tyler Michael Smith committed
Jul 18, 2024 at 19:52 UTC
4ffffccb7eca8a8911ab37b7e15e5d0210838ceb
2 files changed
+40
-21
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
-11
@@ -23,16 +23,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
23
self.is_static_input_scheme = is_static_input_scheme
24
self.cutlass_fp8_supported = cutlass_fp8_supported()
25
26
- # On Lovelace, fail for now if channelwise.
27
- # TODO: (@tms) fallback
28
- if (not self.cutlass_fp8_supported
29
- and self.strategy == QuantizationStrategy.CHANNEL):
30
- raise ValueError(
31
- "Channelwise fp8 quantization requires vLLM's custom "
32
- "cutlass kernels, which are not supported on your device."
33
- "Consider quantizing with per tensor scales or upgrading "
34
- "to Hopper.")
35
-
26
def get_min_capability(self) -> int:
27
# lovelace and up
28
return 89
@@ -53,7 +43,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
43
44
# If channelwise, scales are already lined up, so just transpose.
45
elif self.strategy == QuantizationStrategy.CHANNEL:
56
- assert self.cutlass_fp8_supported
46
weight = layer.weight
47
layer.weight = Parameter(weight.t(), requires_grad=False)
48
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+40
-10
@@ -124,20 +124,50 @@ def apply_fp8_linear(
124
bias=bias)
125
126
else:
127
+ # Note: we pad the input because torch._scaled_mm is more performant
128
+ # for matrices with batch dimension > 16.
129
+ # This could change in the future.
130
qinput, x_scale = ops.scaled_fp8_quant(input,
131
input_scale,
132
batch_dim_padding=17)
133
131
- # Fused GEMM_DQ -- note we padded the input above because
132
- # torch._scaled_mm is more performant for matrices with
133
- # batch dimension > 16. Note that this could change
134
- # in the future.
135
- output, _ = torch._scaled_mm(qinput,
136
- weight,
137
- out_dtype=input.dtype,
138
- scale_a=x_scale,
139
- scale_b=weight_scale,
140
- bias=bias)
134
+ if weight_scale.numel() == 1:
135
+ # Fused GEMM_DQ
136
+ output, _ = torch._scaled_mm(qinput,
137
+ weight,
138
+ out_dtype=input.dtype,
139
+ scale_a=x_scale,
140
+ scale_b=weight_scale,
141
+ bias=bias)
142
+ else:
143
+ # Fallback for channelwise case, where the weight scales are
144
+ # applied separately.
145
+
146
+ # Symmetric quantized GEMM by definition computes the following:
147
+ # C = (s_x * X) (s_w * W) + bias
148
+ # This is equivalent to dequantizing the weights and activations
149
+ # before applying a GEMM.
150
+ #
151
+ # In order to compute quantized operands, a quantized kernel
152
+ # will rewrite the above like so:
153
+ # C = s_w * s_x * (X * W) + bias
154
+ #
155
+ # For the scaled_mm fallback case, we break this down, since it
156
+ # does not support s_w being a vector.
157
+
158
+ # This computes C = sx * (X * W).
159
+ # Output in fp32 to allow subsequent ops to happen in-place
160
+ output, _ = torch._scaled_mm(qinput,
161
+ weight,
162
+ out_dtype=torch.float32,
163
+ scale_a=x_scale)
164
+
165
+ # C = sw * sx * (X * W)
166
+ output = output * weight_scale.t()
167
+ if bias is not None:
168
+ # C = sw * sx * (X * W) + bias
169
+ output = output + bias
170
+ output = output.to(dtype=input.dtype)
171
172
return torch.narrow(output, 0, 0, input.shape[0])
173