@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 226688bd61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][VLM] Make apply_fp8_linear work with >2D input (#9812)

Michael Goin committed Oct 29, 2024 at 22:49 UTC 226688bd6114749633132b9ed074c59d50904830
1 file changed +20 -13
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+20 -13
@@ -96,21 +96,26 @@ def apply_fp8_linear(
96 # If dynamic, layer.input_scale is None and x_scale computed from x.
97 # If static, layer.input_scale is scalar and x_scale is input_scale.
98
99 + # View input as 2D matrix for fp8 methods
100 + input_2d = input.view(-1, input.shape[-1])
101 + output_shape = [*input.shape[:-1], weight.shape[1]]
102 +
103 # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A
104 if cutlass_fp8_supported:
105 qinput, x_scale = ops.scaled_fp8_quant(
102 - input,
106 + input_2d,
107 input_scale,
108 scale_ub=input_scale_ub,
109 use_per_token_if_dynamic=use_per_token_if_dynamic)
110
111 # Fused GEMM_DQ
108 - return ops.cutlass_scaled_mm(qinput,
109 - weight,
110 - out_dtype=input.dtype,
111 - scale_a=x_scale,
112 - scale_b=weight_scale,
113 - bias=bias)
112 + output = ops.cutlass_scaled_mm(qinput,
113 + weight,
114 + out_dtype=input.dtype,
115 + scale_a=x_scale,
116 + scale_b=weight_scale,
117 + bias=bias)
118 + return output.view(*output_shape)
119
120 # torch.scaled_mm supports per tensor weights + activations only
121 # so fallback to naive if per channel or per token
@@ -119,7 +124,7 @@ def apply_fp8_linear(
124 # for matrices with batch dimension > 16.
125 # This could change in the future.
126 qinput, x_scale = ops.scaled_fp8_quant(
122 - input,
127 + input_2d,
128 input_scale,
129 num_token_padding=17,
130 use_per_token_if_dynamic=use_per_token_if_dynamic)
@@ -138,8 +143,10 @@ def apply_fp8_linear(
143 # A fix for discrepancy in scaled_mm which returns tuple
144 # for torch < 2.5 and a single value in torch >= 2.5
145 if type(output) is tuple and len(output) == 2:
141 - return torch.narrow(output[0], 0, 0, input.shape[0])
142 - return torch.narrow(output, 0, 0, input.shape[0])
146 + output = output[0]
147 +
148 + return torch.narrow(output, 0, 0,
149 + input_2d.shape[0]).view(*output_shape)
150
151 else:
152 # Fallback for channelwise case, where we use unfused DQ
@@ -176,15 +183,15 @@ def apply_fp8_linear(
183 if type(output) is tuple and len(output) == 2:
184 output = output[0]
185 # Unpad (undo num_token_padding)
179 - output = torch.narrow(output, 0, 0, input.shape[0])
180 - x_scale = torch.narrow(x_scale, 0, 0, input.shape[0])
186 + output = torch.narrow(output, 0, 0, input_2d.shape[0])
187 + x_scale = torch.narrow(x_scale, 0, 0, input_2d.shape[0])
188
189 # DQ
190 # C = sw * sx * (X * W) + bias
191 output = output * x_scale * weight_scale.t()
192 if bias is not None:
193 output = output + bias
187 - return output.to(dtype=input.dtype)
194 + return output.to(dtype=input.dtype).view(*output_shape)
195
196
197 def apply_int8_linear(