@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c09dade2a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc][Breaking] Change FP8 checkpoint format from act_scale -> input_scale (#5353)

Michael Goin committed Jun 8, 2024 at 13:54 UTC c09dade2a263b6f684d2fbf390c9c1c64761e953
2 files changed +23 -23
vllm/model_executor/layers/quantization/fp8.py
+15 -15
@@ -171,10 +171,10 @@ class Fp8LinearMethod(LinearMethodBase):
171 output_partition_sizes=output_partition_sizes,
172 **extra_weight_attrs)
173
174 - # ACTIVATION SCALE
174 + # INPUT ACTIVATION SCALE
175 if self.quant_config.activation_scheme == "static":
176 self._create_scale_param(
177 - scale_name="act_scale",
177 + scale_name="input_scale",
178 layer=layer,
179 output_partition_sizes=output_partition_sizes,
180 **extra_weight_attrs)
@@ -207,7 +207,7 @@ class Fp8LinearMethod(LinearMethodBase):
207 layer.weight = Parameter(qweight.t(), requires_grad=False)
208 layer.weight_scale = Parameter(weight_scale, requires_grad=False)
209 layer.logical_widths = None
210 - layer.act_scale = None
210 + layer.input_scale = None
211 return
212
213 # If checkpoint is fp8, requantize the separately quantized logical
@@ -232,18 +232,18 @@ class Fp8LinearMethod(LinearMethodBase):
232 weight = layer.weight
233 layer.weight = Parameter(weight.t(), requires_grad=False)
234
235 - # ACT_SCALE
235 + # INPUT ACTIVATION SCALE
236 # Dynamic: set to None (required input to ops.scaled_fp8_quant).
237 - # Static: set to max of the act_scales (since they are equal).
237 + # Static: set to max of the input_scales (since they are equal).
238 if self.quant_config.activation_scheme == "dynamic":
239 - layer.act_scale = None
239 + layer.input_scale = None
240 elif self.quant_config.activation_scheme == "static":
241 - if not all_close_1d(layer.act_scale):
241 + if not all_close_1d(layer.input_scale):
242 raise ValueError(
243 - "All the act_scales for the logical weights of a layer "
244 - f"must be equal. But got {layer.act_scale}")
245 - layer.act_scale = Parameter(layer.act_scale.max(),
246 - requires_grad=False)
243 + "All the input_scales for the logical weights of a "
244 + f"layer must be equal. But got {layer.input_scale}")
245 + layer.input_scale = Parameter(layer.input_scale.max(),
246 + requires_grad=False)
247 else:
248 raise ValueError(
249 f"Unknown scheme {self.quant_config.activation_scheme}")
@@ -254,11 +254,11 @@ class Fp8LinearMethod(LinearMethodBase):
254 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
255
256 # ops.scaled_fp8_quant supports both dynamic and static quant.
257 - # If dynamic, layer.act_scale is None and x_scale computed from x.
258 - # If static, layer.act_scale is scalar and x_scale set to act_scale.
257 + # If dynamic, layer.input_scale is None and x_scale computed from x.
258 + # If static, layer.input_scale is scalar and x_scale is input_scale.
259
260 if bias is None and self.cutlass_fp8_supported:
261 - qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
261 + qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
262
263 # Fused GEMM_DQ
264 output = ops.cutlass_scaled_mm_dq(
@@ -271,7 +271,7 @@ class Fp8LinearMethod(LinearMethodBase):
271
272 else:
273 qinput, x_scale = ops.scaled_fp8_quant(x,
274 - layer.act_scale,
274 + layer.input_scale,
275 batch_dim_padding=17)
276
277 # Fused GEMM_DQ -- note we padded the input above because
vllm/model_executor/models/mixtral.py
+8 -8
@@ -147,7 +147,7 @@ class MixtralMoE(nn.Module):
147 "weight_loader": self.weight_loader,
148 })
149
150 - # ACT_SCALE (for fp8)
150 + # INPUT_SCALE (for fp8)
151 if quant_config.activation_scheme == "static":
152 if not quant_config.is_checkpoint_fp8_serialized:
153 raise ValueError(
@@ -182,11 +182,11 @@ class MixtralMoE(nn.Module):
182 param_data[expert_id, :, :] = loaded_weight[:, shard]
183
184 # Loading scales
185 - if "act_scale" in weight_name or "w2.weight_scale" in weight_name:
185 + if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
186 if param_data[expert_id] != 1 and (param_data[expert_id] -
187 loaded_weight).abs() > 1e-5:
188 raise ValueError(
189 - "act_scales of w1 and w3 of a layer "
189 + "input_scales of w1 and w3 of a layer "
190 f"must be equal. But got {param_data[expert_id]} "
191 f"vs. {loaded_weight}")
192 param_data[expert_id] = loaded_weight
@@ -225,9 +225,9 @@ class MixtralMoE(nn.Module):
225 self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
226
227 else:
228 - # If checkpoint is fp8 + static, cleanup act_scales.
229 - # Since state_dict has an act_scale per expert but our kernels
230 - # are passed one act_scale shared across all experts.
228 + # If checkpoint is fp8 + static, cleanup input_scales.
229 + # Since state_dict has an input_scale per expert but our kernels
230 + # are passed one input_scale shared across all experts.
231 if self.quant_config.activation_scheme == "static":
232 if self.a13_scale is None or self.a2_scale is None:
233 raise ValueError(
@@ -237,7 +237,7 @@ class MixtralMoE(nn.Module):
237 if (not all_close_1d(self.a13_scale)
238 or not all_close_1d(self.a2_scale)):
239 print_warning_once(
240 - "Found act_scales that are not equal for "
240 + "Found input_scales that are not equal for "
241 "fp8 MoE layer. Using the maximum across experts "
242 "for each layer. ")
243
@@ -576,7 +576,7 @@ class MixtralForCausalLM(nn.Module):
576 # These are the activation scales for the experts
577 # (param_name, weight_name, expert_id)
578 ("a13_scale" if weight_name in ["w1", "w3"] else "a2_scale",
579 - f"experts.{expert_id}.{weight_name}.act_scale", expert_id)
579 + f"experts.{expert_id}.{weight_name}.input_scale", expert_id)
580 for expert_id in range(self.config.num_local_experts)
581 for weight_name in ["w1", "w2", "w3"]
582 ]