A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Kernel ] Enable Dynamic Per Token `fp8` (#6547)
Robert Shaw committed
Jul 19, 2024 at 19:08 UTC
4cc24f01b1f1a3e4fc0e740979ebecee0980fbe8
7 files changed
+68
-39
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors -b auto -l 1000 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.769
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.769
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -3,4 +3,5 @@ Meta-Llama-3-8B-Instruct-FP8.yaml
3
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
+Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
tests/kernels/test_fp8_quant.py
+2
-1
@@ -27,7 +27,8 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
27
device="cuda") + 1e-6 # avoid nans
28
29
ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn)
30
- ops_out, ops_scales = ops.dynamic_per_token_scaled_fp8_quant(x)
30
+ ops_out, ops_scales = ops.scaled_fp8_quant(x,
31
+ use_per_token_if_dynamic=True)
32
33
assert torch.allclose(ref_scales, ops_scales)
34
assert torch.allclose(ref_out.to(dtype=torch.float32),
vllm/_custom_ops.py
+13
-13
@@ -300,6 +300,7 @@ def scaled_fp8_quant(
300
input: torch.Tensor,
301
scale: Optional[torch.Tensor] = None,
302
batch_dim_padding: Optional[int] = None,
303
+ use_per_token_if_dynamic: bool = False,
304
) -> Tuple[torch.Tensor, torch.Tensor]:
305
"""
306
Quantize input tensor to FP8 and return quantized tensor and scale.
@@ -315,6 +316,8 @@ def scaled_fp8_quant(
316
scale: Optional scaling factor for the FP8 quantization
317
batch_dim_padding: If specified, pad the first dimension
318
of the output to at least this value.
319
+ use_per_token_if_dynamic: Whether to do per_tensor or per_token
320
+ in the dynamic quantization case.
321
322
Returns:
323
Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
@@ -328,22 +331,19 @@ def scaled_fp8_quant(
331
else:
332
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
333
if scale is None:
331
- scale = torch.zeros(1, device=input.device, dtype=torch.float32)
332
- torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
334
+ if use_per_token_if_dynamic:
335
+ scale = torch.empty((input.numel() // input.shape[-1], 1),
336
+ device=input.device,
337
+ dtype=torch.float32)
338
+ torch.ops._C.dynamic_per_token_scaled_fp8_quant(
339
+ output, input, scale)
340
+ else:
341
+ scale = torch.zeros(1, device=input.device, dtype=torch.float32)
342
+ torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
343
else:
344
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
335
- return output, scale
336
-
345
338
-def dynamic_per_token_scaled_fp8_quant(
339
- input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
340
-
341
- output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
342
- scales = torch.empty((input.numel() // input.shape[-1], 1),
343
- device=input.device,
344
- dtype=torch.float32)
345
- torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
346
- return output, scales
346
+ return output, scale
347
348
349
# int8
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+2
-1
@@ -103,4 +103,5 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
103
weight_scale=layer.weight_scale,
104
input_scale=layer.input_scale,
105
bias=bias,
106
- cutlass_fp8_supported=self.cutlass_fp8_supported)
106
+ cutlass_fp8_supported=self.cutlass_fp8_supported,
107
+ use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/fp8.py
+2
-1
@@ -214,7 +214,8 @@ class Fp8LinearMethod(LinearMethodBase):
214
weight_scale=layer.weight_scale,
215
input_scale=layer.input_scale,
216
bias=bias,
217
- cutlass_fp8_supported=self.cutlass_fp8_supported)
217
+ cutlass_fp8_supported=self.cutlass_fp8_supported,
218
+ use_per_token_if_dynamic=False)
219
220
221
class Fp8MoEMethod(FusedMoEMethodBase):
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+37
-23
@@ -107,31 +107,43 @@ def apply_fp8_linear(
107
input_scale: torch.Tensor,
108
bias: Optional[torch.Tensor] = None,
109
cutlass_fp8_supported: bool = True,
110
+ use_per_token_if_dynamic: bool = False,
111
) -> torch.Tensor:
112
# ops.scaled_fp8_quant supports both dynamic and static quant.
113
# If dynamic, layer.input_scale is None and x_scale computed from x.
114
# If static, layer.input_scale is scalar and x_scale is input_scale.
115
116
+ # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A
117
if cutlass_fp8_supported:
116
- qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
118
+ qinput, x_scale = ops.scaled_fp8_quant(
119
+ input,
120
+ input_scale,
121
+ use_per_token_if_dynamic=use_per_token_if_dynamic)
122
123
# Fused GEMM_DQ
119
- output = ops.cutlass_scaled_mm(qinput,
120
- weight,
121
- out_dtype=input.dtype,
122
- scale_a=x_scale,
123
- scale_b=weight_scale,
124
- bias=bias)
125
-
124
+ return ops.cutlass_scaled_mm(qinput,
125
+ weight,
126
+ out_dtype=input.dtype,
127
+ scale_a=x_scale,
128
+ scale_b=weight_scale,
129
+ bias=bias)
130
+
131
+ # torch.scaled_mm supports per tensor weights + activations only
132
+ # so fallback to naive if per channel or per token
133
else:
134
# Note: we pad the input because torch._scaled_mm is more performant
135
# for matrices with batch dimension > 16.
136
# This could change in the future.
130
- qinput, x_scale = ops.scaled_fp8_quant(input,
131
- input_scale,
132
- batch_dim_padding=17)
137
+ qinput, x_scale = ops.scaled_fp8_quant(
138
+ input,
139
+ input_scale,
140
+ batch_dim_padding=17,
141
+ use_per_token_if_dynamic=use_per_token_if_dynamic)
142
+
143
+ per_tensor_weights = (weight_scale.numel() == 1)
144
+ per_tensor_activations = (x_scale.numel() == 1)
145
134
- if weight_scale.numel() == 1:
146
+ if per_tensor_weights and per_tensor_activations:
147
# Fused GEMM_DQ
148
output, _ = torch._scaled_mm(qinput,
149
weight,
@@ -139,9 +151,11 @@ def apply_fp8_linear(
151
scale_a=x_scale,
152
scale_b=weight_scale,
153
bias=bias)
154
+ return torch.narrow(output, 0, 0, input.shape[0])
155
+
156
else:
143
- # Fallback for channelwise case, where the weight scales are
144
- # applied separately.
157
+ # Fallback for channelwise case, where we use unfused DQ
158
+ # due to limitations with scaled_mm
159
160
# Symmetric quantized GEMM by definition computes the following:
161
# C = (s_x * X) (s_w * W) + bias
@@ -155,21 +169,21 @@ def apply_fp8_linear(
169
# For the scaled_mm fallback case, we break this down, since it
170
# does not support s_w being a vector.
171
158
- # This computes C = sx * (X * W).
172
+ # GEMM
173
+ # This computes C = (X * W).
174
# Output in fp32 to allow subsequent ops to happen in-place
175
output, _ = torch._scaled_mm(qinput,
176
weight,
162
- out_dtype=torch.float32,
163
- scale_a=x_scale)
177
+ out_dtype=torch.float32)
178
+ # Unpad (undo batch_dim_padding)
179
+ output = torch.narrow(output, 0, 0, input.shape[0])
180
165
- # C = sw * sx * (X * W)
166
- output = output * weight_scale.t()
181
+ # DQ
182
+ # C = sw * sx * (X * W) + bias
183
+ output = output * x_scale * weight_scale.t()
184
if bias is not None:
168
- # C = sw * sx * (X * W) + bias
185
output = output + bias
170
- output = output.to(dtype=input.dtype)
171
-
172
- return torch.narrow(output, 0, 0, input.shape[0])
186
+ return output.to(dtype=input.dtype)
187
188
189
def apply_int8_linear(