A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc][Breaking] Change FP8 checkpoint format from act_scale -> input_scale (#5353)
Michael Goin committed
Jun 8, 2024 at 13:54 UTC
c09dade2a263b6f684d2fbf390c9c1c64761e953
2 files changed
+23
-23
vllm/model_executor/layers/quantization/fp8.py
+15
-15
@@ -171,10 +171,10 @@ class Fp8LinearMethod(LinearMethodBase):
171
output_partition_sizes=output_partition_sizes,
172
**extra_weight_attrs)
173
174
- # ACTIVATION SCALE
174
+ # INPUT ACTIVATION SCALE
175
if self.quant_config.activation_scheme == "static":
176
self._create_scale_param(
177
- scale_name="act_scale",
177
+ scale_name="input_scale",
178
layer=layer,
179
output_partition_sizes=output_partition_sizes,
180
**extra_weight_attrs)
@@ -207,7 +207,7 @@ class Fp8LinearMethod(LinearMethodBase):
207
layer.weight = Parameter(qweight.t(), requires_grad=False)
208
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
209
layer.logical_widths = None
210
- layer.act_scale = None
210
+ layer.input_scale = None
211
return
212
213
# If checkpoint is fp8, requantize the separately quantized logical
@@ -232,18 +232,18 @@ class Fp8LinearMethod(LinearMethodBase):
232
weight = layer.weight
233
layer.weight = Parameter(weight.t(), requires_grad=False)
234
235
- # ACT_SCALE
235
+ # INPUT ACTIVATION SCALE
236
# Dynamic: set to None (required input to ops.scaled_fp8_quant).
237
- # Static: set to max of the act_scales (since they are equal).
237
+ # Static: set to max of the input_scales (since they are equal).
238
if self.quant_config.activation_scheme == "dynamic":
239
- layer.act_scale = None
239
+ layer.input_scale = None
240
elif self.quant_config.activation_scheme == "static":
241
- if not all_close_1d(layer.act_scale):
241
+ if not all_close_1d(layer.input_scale):
242
raise ValueError(
243
- "All the act_scales for the logical weights of a layer "
244
- f"must be equal. But got {layer.act_scale}")
245
- layer.act_scale = Parameter(layer.act_scale.max(),
246
- requires_grad=False)
243
+ "All the input_scales for the logical weights of a "
244
+ f"layer must be equal. But got {layer.input_scale}")
245
+ layer.input_scale = Parameter(layer.input_scale.max(),
246
+ requires_grad=False)
247
else:
248
raise ValueError(
249
f"Unknown scheme {self.quant_config.activation_scheme}")
@@ -254,11 +254,11 @@ class Fp8LinearMethod(LinearMethodBase):
254
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
255
256
# ops.scaled_fp8_quant supports both dynamic and static quant.
257
- # If dynamic, layer.act_scale is None and x_scale computed from x.
258
- # If static, layer.act_scale is scalar and x_scale set to act_scale.
257
+ # If dynamic, layer.input_scale is None and x_scale computed from x.
258
+ # If static, layer.input_scale is scalar and x_scale is input_scale.
259
260
if bias is None and self.cutlass_fp8_supported:
261
- qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
261
+ qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
262
263
# Fused GEMM_DQ
264
output = ops.cutlass_scaled_mm_dq(
@@ -271,7 +271,7 @@ class Fp8LinearMethod(LinearMethodBase):
271
272
else:
273
qinput, x_scale = ops.scaled_fp8_quant(x,
274
- layer.act_scale,
274
+ layer.input_scale,
275
batch_dim_padding=17)
276
277
# Fused GEMM_DQ -- note we padded the input above because
vllm/model_executor/models/mixtral.py
+8
-8
@@ -147,7 +147,7 @@ class MixtralMoE(nn.Module):
147
"weight_loader": self.weight_loader,
148
})
149
150
- # ACT_SCALE (for fp8)
150
+ # INPUT_SCALE (for fp8)
151
if quant_config.activation_scheme == "static":
152
if not quant_config.is_checkpoint_fp8_serialized:
153
raise ValueError(
@@ -182,11 +182,11 @@ class MixtralMoE(nn.Module):
182
param_data[expert_id, :, :] = loaded_weight[:, shard]
183
184
# Loading scales
185
- if "act_scale" in weight_name or "w2.weight_scale" in weight_name:
185
+ if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
186
if param_data[expert_id] != 1 and (param_data[expert_id] -
187
loaded_weight).abs() > 1e-5:
188
raise ValueError(
189
- "act_scales of w1 and w3 of a layer "
189
+ "input_scales of w1 and w3 of a layer "
190
f"must be equal. But got {param_data[expert_id]} "
191
f"vs. {loaded_weight}")
192
param_data[expert_id] = loaded_weight
@@ -225,9 +225,9 @@ class MixtralMoE(nn.Module):
225
self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
226
227
else:
228
- # If checkpoint is fp8 + static, cleanup act_scales.
229
- # Since state_dict has an act_scale per expert but our kernels
230
- # are passed one act_scale shared across all experts.
228
+ # If checkpoint is fp8 + static, cleanup input_scales.
229
+ # Since state_dict has an input_scale per expert but our kernels
230
+ # are passed one input_scale shared across all experts.
231
if self.quant_config.activation_scheme == "static":
232
if self.a13_scale is None or self.a2_scale is None:
233
raise ValueError(
@@ -237,7 +237,7 @@ class MixtralMoE(nn.Module):
237
if (not all_close_1d(self.a13_scale)
238
or not all_close_1d(self.a2_scale)):
239
print_warning_once(
240
- "Found act_scales that are not equal for "
240
+ "Found input_scales that are not equal for "
241
"fp8 MoE layer. Using the maximum across experts "
242
"for each layer. ")
243
@@ -576,7 +576,7 @@ class MixtralForCausalLM(nn.Module):
576
# These are the activation scales for the experts
577
# (param_name, weight_name, expert_id)
578
("a13_scale" if weight_name in ["w1", "w3"] else "a2_scale",
579
- f"experts.{expert_id}.{weight_name}.act_scale", expert_id)
579
+ f"experts.{expert_id}.{weight_name}.input_scale", expert_id)
580
for expert_id in range(self.config.num_local_experts)
581
for weight_name in ["w1", "w2", "w3"]
582
]