@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5563a4dea8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] Correct Mixtral FP8 checkpoint loading (#5231)

Cody Yu committed Jun 5, 2024 at 10:58 UTC 5563a4dea86e62a560c8c99537ce614d5de894e0
2 files changed +80 -35
vllm/model_executor/layers/quantization/fp8.py
+4 -3
@@ -300,14 +300,15 @@ def all_close_1d(x: torch.Tensor) -> bool:
300
301
302 def per_tensor_quantize(tensor: torch.Tensor,
303 - inv_scale: float) -> torch.Tensor:
303 + inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
304 finfo = torch.finfo(torch.float8_e4m3fn)
305 qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
306 return qweight.to(torch.float8_e4m3fn)
307
308
309 -def per_tensor_dequantize(tensor: torch.Tensor,
310 - inv_scale: float) -> torch.Tensor:
309 +def per_tensor_dequantize(
310 + tensor: torch.Tensor, inv_scale: Union[float,
311 + torch.Tensor]) -> torch.Tensor:
312 fake_qweight = tensor.to(torch.float16)
313 dq_weight = fake_qweight * inv_scale
314 return dq_weight
vllm/model_executor/models/mixtral.py
+76 -32
@@ -41,7 +41,9 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization.base_config import (
43 QuantizationConfig)
44 -from vllm.model_executor.layers.quantization.fp8 import Fp8Config
44 +from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,
45 + per_tensor_dequantize,
46 + per_tensor_quantize)
47 from vllm.model_executor.layers.rotary_embedding import get_rope
48 from vllm.model_executor.layers.sampler import Sampler
49 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -98,16 +100,16 @@ class MixtralMoE(nn.Module):
100 if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
101 params_dtype = torch.float8_e4m3fn
102
101 - self.w13_weight = nn.Parameter(
102 - torch.empty(self.num_total_experts,
103 - 2 * self.intermediate_size,
104 - self.hidden_size,
105 - dtype=params_dtype))
106 - self.w2_weight = nn.Parameter(
107 - torch.empty(self.num_total_experts,
108 - self.hidden_size,
109 - self.intermediate_size,
110 - dtype=params_dtype))
103 + self.w13_weight = nn.Parameter(torch.empty(self.num_total_experts,
104 + 2 * self.intermediate_size,
105 + self.hidden_size,
106 + dtype=params_dtype),
107 + requires_grad=False)
108 + self.w2_weight = nn.Parameter(torch.empty(self.num_total_experts,
109 + self.hidden_size,
110 + self.intermediate_size,
111 + dtype=params_dtype),
112 + requires_grad=False)
113
114 set_weight_attrs(self.w13_weight, {
115 "weight_loader": self.weight_loader,
@@ -124,7 +126,10 @@ class MixtralMoE(nn.Module):
126
127 if self.use_fp8:
128 # WEIGHT_SCALE (for fp8)
129 + # Allocate 2 scales for w1 and w3 respectively.
130 + # They will be combined to a single scale after weight loading.
131 self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
132 + 2,
133 dtype=torch.float32),
134 requires_grad=False)
135 self.w2_scale = nn.Parameter(torch.ones(self.num_total_experts,
@@ -148,11 +153,11 @@ class MixtralMoE(nn.Module):
153 raise ValueError(
154 "Found static activation scheme for checkpoint that "
155 "was not serialized fp8.")
151 - self.a13_scale = nn.Parameter(torch.zeros(
156 + self.a13_scale = nn.Parameter(torch.ones(
157 self.num_total_experts, dtype=torch.float32),
158 requires_grad=False)
154 - self.a2_scale = nn.Parameter(torch.zeros(
155 - self.num_total_experts, dtype=torch.float32),
159 + self.a2_scale = nn.Parameter(torch.ones(self.num_total_experts,
160 + dtype=torch.float32),
161 requires_grad=False)
162
163 set_weight_attrs(self.a13_scale, {
@@ -175,8 +180,22 @@ class MixtralMoE(nn.Module):
180 shard_size:2 * shard_size, :] = loaded_weight[shard, :]
181 if weight_name.endswith("w2.weight"):
182 param_data[expert_id, :, :] = loaded_weight[:, shard]
178 - if "act_scale" in weight_name or "weight_scale" in weight_name:
183 +
184 + # Loading scales
185 + if "act_scale" in weight_name or "w2.weight_scale" in weight_name:
186 + if param_data[expert_id] != 1 and (param_data[expert_id] -
187 + loaded_weight).abs() > 1e-5:
188 + raise ValueError(
189 + "act_scales of w1 and w3 of a layer "
190 + f"must be equal. But got {param_data[expert_id]} "
191 + f"vs. {loaded_weight}")
192 param_data[expert_id] = loaded_weight
193 + elif "weight_scale" in weight_name:
194 + # We have to keep the weight scales of w1 and w3 because
195 + # we need to re-quantize w1/w3 weights after weight loading.
196 + assert "w1" in weight_name or "w3" in weight_name
197 + shard_id = 0 if "w1" in weight_name else 1
198 + param_data[expert_id][shard_id] = loaded_weight
199
200 def process_weights_after_loading(self):
201 # Fp8 is the only case where we need to process after loading.
@@ -189,6 +208,12 @@ class MixtralMoE(nn.Module):
208 dtype=torch.float8_e4m3fn)
209 w2_weight = torch.empty_like(self.w2_weight.data,
210 dtype=torch.float8_e4m3fn)
211 +
212 + # Re-initialize w13_scale because we directly quantize
213 + # merged w13 weights and generate a single scaling factor.
214 + self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
215 + dtype=torch.float32),
216 + requires_grad=False)
217 for expert in range(self.num_total_experts):
218 w13_weight[expert, :, :], self.w13_scale[
219 expert] = ops.scaled_fp8_quant(
@@ -199,25 +224,44 @@ class MixtralMoE(nn.Module):
224 self.w13_weight = nn.Parameter(w13_weight, requires_grad=False)
225 self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
226
202 - # If checkpoint is fp8 + static, cleanup act_scales.
203 - # Since state_dict has an act_scale per expert but our kernels
204 - # are passed one act_scale shared across all experts.
205 - elif self.quant_config.activation_scheme == "static":
206 - if self.a13_scale is None or self.a2_scale is None:
207 - raise ValueError(
208 - "QuantConfig has static quantization, but found "
209 - "activation scales are None.")
227 + else:
228 + # If checkpoint is fp8 + static, cleanup act_scales.
229 + # Since state_dict has an act_scale per expert but our kernels
230 + # are passed one act_scale shared across all experts.
231 + if self.quant_config.activation_scheme == "static":
232 + if self.a13_scale is None or self.a2_scale is None:
233 + raise ValueError(
234 + "QuantConfig has static quantization, but found "
235 + "activation scales are None.")
236
211 - if (not all_close_1d(self.a13_scale)
212 - or not all_close_1d(self.a2_scale)):
213 - print_warning_once(
214 - "Found act_scales that are not equal for fp8 MoE layer. "
215 - "Using the maximum across experts for each layer. ")
237 + if (not all_close_1d(self.a13_scale)
238 + or not all_close_1d(self.a2_scale)):
239 + print_warning_once(
240 + "Found act_scales that are not equal for "
241 + "fp8 MoE layer. Using the maximum across experts "
242 + "for each layer. ")
243
217 - self.a13_scale = nn.Parameter(self.a13_scale.max(),
218 - requires_grad=False)
219 - self.a2_scale = nn.Parameter(self.a2_scale.max(),
220 - requires_grad=False)
244 + self.a13_scale = nn.Parameter(self.a13_scale.max(),
245 + requires_grad=False)
246 + self.a2_scale = nn.Parameter(self.a2_scale.max(),
247 + requires_grad=False)
248 +
249 + assert self.w13_scale is not None
250 + shard_size = self.intermediate_size
251 + max_w13_scales = self.w13_scale.max(dim=1).values
252 + for expert_id in range(self.num_total_experts):
253 + start = 0
254 + for shard_id in range(2):
255 + dq_weight = per_tensor_dequantize(
256 + self.w13_weight[expert_id][start:start +
257 + shard_size, :],
258 + self.w13_scale[expert_id][shard_id])
259 + self.w13_weight[expert_id][
260 + start:start + shard_size, :] = per_tensor_quantize(
261 + dq_weight, max_w13_scales[expert_id])
262 + start += shard_size
263 +
264 + self.w13_scale = nn.Parameter(max_w13_scales, requires_grad=False)
265
266 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
267 num_tokens, hidden_size = hidden_states.shape