A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Correct Mixtral FP8 checkpoint loading (#5231)
Cody Yu committed
Jun 5, 2024 at 10:58 UTC
5563a4dea86e62a560c8c99537ce614d5de894e0
2 files changed
+80
-35
vllm/model_executor/layers/quantization/fp8.py
+4
-3
@@ -300,14 +300,15 @@ def all_close_1d(x: torch.Tensor) -> bool:
300
301
302
def per_tensor_quantize(tensor: torch.Tensor,
303
- inv_scale: float) -> torch.Tensor:
303
+ inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
304
finfo = torch.finfo(torch.float8_e4m3fn)
305
qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
306
return qweight.to(torch.float8_e4m3fn)
307
308
309
-def per_tensor_dequantize(tensor: torch.Tensor,
310
- inv_scale: float) -> torch.Tensor:
309
+def per_tensor_dequantize(
310
+ tensor: torch.Tensor, inv_scale: Union[float,
311
+ torch.Tensor]) -> torch.Tensor:
312
fake_qweight = tensor.to(torch.float16)
313
dq_weight = fake_qweight * inv_scale
314
return dq_weight
vllm/model_executor/models/mixtral.py
+76
-32
@@ -41,7 +41,9 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization.base_config import (
43
QuantizationConfig)
44
-from vllm.model_executor.layers.quantization.fp8 import Fp8Config
44
+from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,
45
+ per_tensor_dequantize,
46
+ per_tensor_quantize)
47
from vllm.model_executor.layers.rotary_embedding import get_rope
48
from vllm.model_executor.layers.sampler import Sampler
49
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -98,16 +100,16 @@ class MixtralMoE(nn.Module):
100
if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
101
params_dtype = torch.float8_e4m3fn
102
101
- self.w13_weight = nn.Parameter(
102
- torch.empty(self.num_total_experts,
103
- 2 * self.intermediate_size,
104
- self.hidden_size,
105
- dtype=params_dtype))
106
- self.w2_weight = nn.Parameter(
107
- torch.empty(self.num_total_experts,
108
- self.hidden_size,
109
- self.intermediate_size,
110
- dtype=params_dtype))
103
+ self.w13_weight = nn.Parameter(torch.empty(self.num_total_experts,
104
+ 2 * self.intermediate_size,
105
+ self.hidden_size,
106
+ dtype=params_dtype),
107
+ requires_grad=False)
108
+ self.w2_weight = nn.Parameter(torch.empty(self.num_total_experts,
109
+ self.hidden_size,
110
+ self.intermediate_size,
111
+ dtype=params_dtype),
112
+ requires_grad=False)
113
114
set_weight_attrs(self.w13_weight, {
115
"weight_loader": self.weight_loader,
@@ -124,7 +126,10 @@ class MixtralMoE(nn.Module):
126
127
if self.use_fp8:
128
# WEIGHT_SCALE (for fp8)
129
+ # Allocate 2 scales for w1 and w3 respectively.
130
+ # They will be combined to a single scale after weight loading.
131
self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
132
+ 2,
133
dtype=torch.float32),
134
requires_grad=False)
135
self.w2_scale = nn.Parameter(torch.ones(self.num_total_experts,
@@ -148,11 +153,11 @@ class MixtralMoE(nn.Module):
153
raise ValueError(
154
"Found static activation scheme for checkpoint that "
155
"was not serialized fp8.")
151
- self.a13_scale = nn.Parameter(torch.zeros(
156
+ self.a13_scale = nn.Parameter(torch.ones(
157
self.num_total_experts, dtype=torch.float32),
158
requires_grad=False)
154
- self.a2_scale = nn.Parameter(torch.zeros(
155
- self.num_total_experts, dtype=torch.float32),
159
+ self.a2_scale = nn.Parameter(torch.ones(self.num_total_experts,
160
+ dtype=torch.float32),
161
requires_grad=False)
162
163
set_weight_attrs(self.a13_scale, {
@@ -175,8 +180,22 @@ class MixtralMoE(nn.Module):
180
shard_size:2 * shard_size, :] = loaded_weight[shard, :]
181
if weight_name.endswith("w2.weight"):
182
param_data[expert_id, :, :] = loaded_weight[:, shard]
178
- if "act_scale" in weight_name or "weight_scale" in weight_name:
183
+
184
+ # Loading scales
185
+ if "act_scale" in weight_name or "w2.weight_scale" in weight_name:
186
+ if param_data[expert_id] != 1 and (param_data[expert_id] -
187
+ loaded_weight).abs() > 1e-5:
188
+ raise ValueError(
189
+ "act_scales of w1 and w3 of a layer "
190
+ f"must be equal. But got {param_data[expert_id]} "
191
+ f"vs. {loaded_weight}")
192
param_data[expert_id] = loaded_weight
193
+ elif "weight_scale" in weight_name:
194
+ # We have to keep the weight scales of w1 and w3 because
195
+ # we need to re-quantize w1/w3 weights after weight loading.
196
+ assert "w1" in weight_name or "w3" in weight_name
197
+ shard_id = 0 if "w1" in weight_name else 1
198
+ param_data[expert_id][shard_id] = loaded_weight
199
200
def process_weights_after_loading(self):
201
# Fp8 is the only case where we need to process after loading.
@@ -189,6 +208,12 @@ class MixtralMoE(nn.Module):
208
dtype=torch.float8_e4m3fn)
209
w2_weight = torch.empty_like(self.w2_weight.data,
210
dtype=torch.float8_e4m3fn)
211
+
212
+ # Re-initialize w13_scale because we directly quantize
213
+ # merged w13 weights and generate a single scaling factor.
214
+ self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
215
+ dtype=torch.float32),
216
+ requires_grad=False)
217
for expert in range(self.num_total_experts):
218
w13_weight[expert, :, :], self.w13_scale[
219
expert] = ops.scaled_fp8_quant(
@@ -199,25 +224,44 @@ class MixtralMoE(nn.Module):
224
self.w13_weight = nn.Parameter(w13_weight, requires_grad=False)
225
self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
226
202
- # If checkpoint is fp8 + static, cleanup act_scales.
203
- # Since state_dict has an act_scale per expert but our kernels
204
- # are passed one act_scale shared across all experts.
205
- elif self.quant_config.activation_scheme == "static":
206
- if self.a13_scale is None or self.a2_scale is None:
207
- raise ValueError(
208
- "QuantConfig has static quantization, but found "
209
- "activation scales are None.")
227
+ else:
228
+ # If checkpoint is fp8 + static, cleanup act_scales.
229
+ # Since state_dict has an act_scale per expert but our kernels
230
+ # are passed one act_scale shared across all experts.
231
+ if self.quant_config.activation_scheme == "static":
232
+ if self.a13_scale is None or self.a2_scale is None:
233
+ raise ValueError(
234
+ "QuantConfig has static quantization, but found "
235
+ "activation scales are None.")
236
211
- if (not all_close_1d(self.a13_scale)
212
- or not all_close_1d(self.a2_scale)):
213
- print_warning_once(
214
- "Found act_scales that are not equal for fp8 MoE layer. "
215
- "Using the maximum across experts for each layer. ")
237
+ if (not all_close_1d(self.a13_scale)
238
+ or not all_close_1d(self.a2_scale)):
239
+ print_warning_once(
240
+ "Found act_scales that are not equal for "
241
+ "fp8 MoE layer. Using the maximum across experts "
242
+ "for each layer. ")
243
217
- self.a13_scale = nn.Parameter(self.a13_scale.max(),
218
- requires_grad=False)
219
- self.a2_scale = nn.Parameter(self.a2_scale.max(),
220
- requires_grad=False)
244
+ self.a13_scale = nn.Parameter(self.a13_scale.max(),
245
+ requires_grad=False)
246
+ self.a2_scale = nn.Parameter(self.a2_scale.max(),
247
+ requires_grad=False)
248
+
249
+ assert self.w13_scale is not None
250
+ shard_size = self.intermediate_size
251
+ max_w13_scales = self.w13_scale.max(dim=1).values
252
+ for expert_id in range(self.num_total_experts):
253
+ start = 0
254
+ for shard_id in range(2):
255
+ dq_weight = per_tensor_dequantize(
256
+ self.w13_weight[expert_id][start:start +
257
+ shard_size, :],
258
+ self.w13_scale[expert_id][shard_id])
259
+ self.w13_weight[expert_id][
260
+ start:start + shard_size, :] = per_tensor_quantize(
261
+ dq_weight, max_w13_scales[expert_id])
262
+ start += shard_size
263
+
264
+ self.w13_scale = nn.Parameter(max_w13_scales, requires_grad=False)
265
266
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
267
num_tokens, hidden_size = hidden_states.shape