@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4cc24f01b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Kernel ] Enable Dynamic Per Token `fp8` (#6547)

Robert Shaw committed Jul 19, 2024 at 19:08 UTC 4cc24f01b1f1a3e4fc0e740979ebecee0980fbe8
7 files changed +68 -39
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors -b auto -l 1000 -f 5 -t 1
2 +model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.769
8 + - name: "exact_match,flexible-extract"
9 + value: 0.769
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -3,4 +3,5 @@ Meta-Llama-3-8B-Instruct-FP8.yaml
3 Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 +Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7 Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
tests/kernels/test_fp8_quant.py
+2 -1
@@ -27,7 +27,8 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
27 device="cuda") + 1e-6 # avoid nans
28
29 ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.float8_e4m3fn)
30 - ops_out, ops_scales = ops.dynamic_per_token_scaled_fp8_quant(x)
30 + ops_out, ops_scales = ops.scaled_fp8_quant(x,
31 + use_per_token_if_dynamic=True)
32
33 assert torch.allclose(ref_scales, ops_scales)
34 assert torch.allclose(ref_out.to(dtype=torch.float32),
vllm/_custom_ops.py
+13 -13
@@ -300,6 +300,7 @@ def scaled_fp8_quant(
300 input: torch.Tensor,
301 scale: Optional[torch.Tensor] = None,
302 batch_dim_padding: Optional[int] = None,
303 + use_per_token_if_dynamic: bool = False,
304 ) -> Tuple[torch.Tensor, torch.Tensor]:
305 """
306 Quantize input tensor to FP8 and return quantized tensor and scale.
@@ -315,6 +316,8 @@ def scaled_fp8_quant(
316 scale: Optional scaling factor for the FP8 quantization
317 batch_dim_padding: If specified, pad the first dimension
318 of the output to at least this value.
319 + use_per_token_if_dynamic: Whether to do per_tensor or per_token
320 + in the dynamic quantization case.
321
322 Returns:
323 Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
@@ -328,22 +331,19 @@ def scaled_fp8_quant(
331 else:
332 output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
333 if scale is None:
331 - scale = torch.zeros(1, device=input.device, dtype=torch.float32)
332 - torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
334 + if use_per_token_if_dynamic:
335 + scale = torch.empty((input.numel() // input.shape[-1], 1),
336 + device=input.device,
337 + dtype=torch.float32)
338 + torch.ops._C.dynamic_per_token_scaled_fp8_quant(
339 + output, input, scale)
340 + else:
341 + scale = torch.zeros(1, device=input.device, dtype=torch.float32)
342 + torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
343 else:
344 torch.ops._C.static_scaled_fp8_quant(output, input, scale)
335 - return output, scale
336 -
345
338 -def dynamic_per_token_scaled_fp8_quant(
339 - input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
340 -
341 - output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
342 - scales = torch.empty((input.numel() // input.shape[-1], 1),
343 - device=input.device,
344 - dtype=torch.float32)
345 - torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
346 - return output, scales
346 + return output, scale
347
348
349 # int8
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+2 -1
@@ -103,4 +103,5 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
103 weight_scale=layer.weight_scale,
104 input_scale=layer.input_scale,
105 bias=bias,
106 - cutlass_fp8_supported=self.cutlass_fp8_supported)
106 + cutlass_fp8_supported=self.cutlass_fp8_supported,
107 + use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/fp8.py
+2 -1
@@ -214,7 +214,8 @@ class Fp8LinearMethod(LinearMethodBase):
214 weight_scale=layer.weight_scale,
215 input_scale=layer.input_scale,
216 bias=bias,
217 - cutlass_fp8_supported=self.cutlass_fp8_supported)
217 + cutlass_fp8_supported=self.cutlass_fp8_supported,
218 + use_per_token_if_dynamic=False)
219
220
221 class Fp8MoEMethod(FusedMoEMethodBase):
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+37 -23
@@ -107,31 +107,43 @@ def apply_fp8_linear(
107 input_scale: torch.Tensor,
108 bias: Optional[torch.Tensor] = None,
109 cutlass_fp8_supported: bool = True,
110 + use_per_token_if_dynamic: bool = False,
111 ) -> torch.Tensor:
112 # ops.scaled_fp8_quant supports both dynamic and static quant.
113 # If dynamic, layer.input_scale is None and x_scale computed from x.
114 # If static, layer.input_scale is scalar and x_scale is input_scale.
115
116 + # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A
117 if cutlass_fp8_supported:
116 - qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
118 + qinput, x_scale = ops.scaled_fp8_quant(
119 + input,
120 + input_scale,
121 + use_per_token_if_dynamic=use_per_token_if_dynamic)
122
123 # Fused GEMM_DQ
119 - output = ops.cutlass_scaled_mm(qinput,
120 - weight,
121 - out_dtype=input.dtype,
122 - scale_a=x_scale,
123 - scale_b=weight_scale,
124 - bias=bias)
125 -
124 + return ops.cutlass_scaled_mm(qinput,
125 + weight,
126 + out_dtype=input.dtype,
127 + scale_a=x_scale,
128 + scale_b=weight_scale,
129 + bias=bias)
130 +
131 + # torch.scaled_mm supports per tensor weights + activations only
132 + # so fallback to naive if per channel or per token
133 else:
134 # Note: we pad the input because torch._scaled_mm is more performant
135 # for matrices with batch dimension > 16.
136 # This could change in the future.
130 - qinput, x_scale = ops.scaled_fp8_quant(input,
131 - input_scale,
132 - batch_dim_padding=17)
137 + qinput, x_scale = ops.scaled_fp8_quant(
138 + input,
139 + input_scale,
140 + batch_dim_padding=17,
141 + use_per_token_if_dynamic=use_per_token_if_dynamic)
142 +
143 + per_tensor_weights = (weight_scale.numel() == 1)
144 + per_tensor_activations = (x_scale.numel() == 1)
145
134 - if weight_scale.numel() == 1:
146 + if per_tensor_weights and per_tensor_activations:
147 # Fused GEMM_DQ
148 output, _ = torch._scaled_mm(qinput,
149 weight,
@@ -139,9 +151,11 @@ def apply_fp8_linear(
151 scale_a=x_scale,
152 scale_b=weight_scale,
153 bias=bias)
154 + return torch.narrow(output, 0, 0, input.shape[0])
155 +
156 else:
143 - # Fallback for channelwise case, where the weight scales are
144 - # applied separately.
157 + # Fallback for channelwise case, where we use unfused DQ
158 + # due to limitations with scaled_mm
159
160 # Symmetric quantized GEMM by definition computes the following:
161 # C = (s_x * X) (s_w * W) + bias
@@ -155,21 +169,21 @@ def apply_fp8_linear(
169 # For the scaled_mm fallback case, we break this down, since it
170 # does not support s_w being a vector.
171
158 - # This computes C = sx * (X * W).
172 + # GEMM
173 + # This computes C = (X * W).
174 # Output in fp32 to allow subsequent ops to happen in-place
175 output, _ = torch._scaled_mm(qinput,
176 weight,
162 - out_dtype=torch.float32,
163 - scale_a=x_scale)
177 + out_dtype=torch.float32)
178 + # Unpad (undo batch_dim_padding)
179 + output = torch.narrow(output, 0, 0, input.shape[0])
180
165 - # C = sw * sx * (X * W)
166 - output = output * weight_scale.t()
181 + # DQ
182 + # C = sw * sx * (X * W) + bias
183 + output = output * x_scale * weight_scale.t()
184 if bias is not None:
168 - # C = sw * sx * (X * W) + bias
185 output = output + bias
170 - output = output.to(dtype=input.dtype)
171 -
172 - return torch.narrow(output, 0, 0, input.shape[0])
186 + return output.to(dtype=input.dtype)
187
188
189 def apply_int8_linear(