A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc] Support FP8 MoE for compressed-tensors (#8588)
Michael Goin committed
Sep 25, 2024 at 12:43 UTC
873edda6cf8a2902e8b08eea0bf8f8f6d73704a8
5 files changed
+226
-8
tests/weight_loading/models-large.txt
+1
@@ -1,4 +1,5 @@
1
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4
+compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
5
gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
vllm/model_executor/layers/fused_moe/layer.py
+7
-2
@@ -323,10 +323,12 @@ class FusedMoE(torch.nn.Module):
323
loaded_weight: torch.Tensor, weight_name: str,
324
shard_id: str, expert_id: int) -> None:
325
326
- # compressed-tensors represents weights on disk which are flipped
326
+ # compressed-tensors checkpoints with packed weights are stored flipped
327
+ # TODO (mgoin): check self.quant_method.quant_config.quant_format
328
+ # against known CompressionFormat enum values that have this quality
329
loaded_weight = loaded_weight.t().contiguous() if (
330
self.quant_method.__class__.__name__
329
- == "CompressedTensorsMoEMethod") else loaded_weight
331
+ == "CompressedTensorsWNA16MoEMethod") else loaded_weight
332
333
if shard_id not in ("w1", "w2", "w3"):
334
raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
@@ -353,6 +355,9 @@ class FusedMoE(torch.nn.Module):
355
356
# Case input scale: input_scale loading is only supported for fp8
357
if "input_scale" in weight_name:
358
+ # this is needed for compressed-tensors only
359
+ loaded_weight = loaded_weight.to(param.data.device)
360
+
361
if param.data[expert_id] != 1 and (param.data[expert_id] -
362
loaded_weight).abs() > 1e-5:
363
raise ValueError(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1
-1
@@ -73,7 +73,7 @@ class CompressedTensorsConfig(QuantizationConfig):
73
if isinstance(layer, Attention):
74
return CompressedTensorsKVCacheMethod(self)
75
if isinstance(layer, FusedMoE):
76
- return CompressedTensorsMoEMethod(self)
76
+ return CompressedTensorsMoEMethod.get_moe_method(self)
77
return None
78
79
@classmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+215
-3
@@ -5,12 +5,16 @@ from typing import Callable, List, Optional
5
import torch
6
7
from vllm import _custom_ops as ops
8
-from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
8
+from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
9
+ FusedMoeWeightScaleSupported)
10
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
11
WNA16_SUPPORTED_BITS)
12
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12
- CompressionFormat)
13
+ CompressionFormat, QuantizationStrategy)
14
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
15
+ all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
16
from vllm.model_executor.utils import set_weight_attrs
17
+from vllm.utils import is_hip, print_warning_once
18
19
20
class GPTQMarlinState(Enum):
@@ -18,11 +22,219 @@ class GPTQMarlinState(Enum):
22
READY = enum.auto()
23
24
21
-__all__ = ["CompressedTensorsMoEMethod"]
25
+__all__ = [
26
+ "CompressedTensorsMoEMethod", "CompressedTensorsW8A8Fp8MoEMethod",
27
+ "CompressedTensorsWNA16MoEMethod"
28
+]
29
30
31
class CompressedTensorsMoEMethod(FusedMoEMethodBase):
32
33
+ @staticmethod
34
+ def get_moe_method(
35
+ quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
36
+ ) -> "CompressedTensorsMoEMethod":
37
+ # TODO: @dsikka: refactor this to use schemes as other kernels
38
+ # are supported + check if the layer is being ignored.
39
+ weight_quant = quant_config.target_scheme_map["Linear"].get("weights")
40
+ input_quant = quant_config.target_scheme_map["Linear"].get(
41
+ "input_activations")
42
+
43
+ if quant_config._is_wNa16_group_channel(weight_quant, input_quant):
44
+ return CompressedTensorsWNA16MoEMethod(quant_config)
45
+ elif quant_config._is_fp8_w8a8(weight_quant, input_quant):
46
+ return CompressedTensorsW8A8Fp8MoEMethod(quant_config)
47
+ else:
48
+ raise RuntimeError(
49
+ f"Unsupported FusedMoe scheme: {weight_quant}, {input_quant}")
50
+
51
+
52
+class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
53
+
54
+ def __init__(
55
+ self,
56
+ quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
57
+ ):
58
+ self.quant_config = quant_config
59
+ self.weight_quant = self.quant_config.target_scheme_map["Linear"].get(
60
+ "weights")
61
+ self.input_quant = self.quant_config.target_scheme_map["Linear"].get(
62
+ "input_activations")
63
+
64
+ if not (self.weight_quant.strategy == QuantizationStrategy.TENSOR
65
+ and self.input_quant.strategy == QuantizationStrategy.TENSOR):
66
+ raise ValueError(
67
+ "For FP8 Fused MoE layers, only per-tensor scales"
68
+ "for weights and activations are supported. Found "
69
+ f"{self.weight_quant}, {self.input_quant}")
70
+
71
+ self.static_input_scales = not self.input_quant.dynamic
72
+
73
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
74
+ hidden_size: int, intermediate_size: int,
75
+ params_dtype: torch.dtype, **extra_weight_attrs):
76
+
77
+ params_dtype = torch.float8_e4m3fn
78
+
79
+ # WEIGHTS
80
+ w13_weight = torch.nn.Parameter(torch.empty(num_experts,
81
+ 2 * intermediate_size,
82
+ hidden_size,
83
+ dtype=params_dtype),
84
+ requires_grad=False)
85
+ layer.register_parameter("w13_weight", w13_weight)
86
+ set_weight_attrs(w13_weight, extra_weight_attrs)
87
+
88
+ w2_weight = torch.nn.Parameter(torch.empty(num_experts,
89
+ hidden_size,
90
+ intermediate_size,
91
+ dtype=params_dtype),
92
+ requires_grad=False)
93
+ layer.register_parameter("w2_weight", w2_weight)
94
+ set_weight_attrs(w2_weight, extra_weight_attrs)
95
+
96
+ # WEIGHT_SCALES
97
+ # Allocate 2 scales for w1 and w3 respectively.
98
+ # They will be combined to a single scale after weight loading.
99
+ w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
100
+ 2,
101
+ dtype=torch.float32),
102
+ requires_grad=False)
103
+ layer.register_parameter("w13_weight_scale", w13_weight_scale)
104
+
105
+ w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
106
+ dtype=torch.float32),
107
+ requires_grad=False)
108
+ layer.register_parameter("w2_weight_scale", w2_weight_scale)
109
+ # Add the quantization method used (per tensor/grouped/channel)
110
+ # to ensure the weight scales are loaded in properly
111
+ extra_weight_attrs.update(
112
+ {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
113
+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)
114
+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)
115
+
116
+ # INPUT_SCALES
117
+ if self.static_input_scales:
118
+ w13_input_scale = torch.nn.Parameter(torch.ones(
119
+ num_experts, dtype=torch.float32),
120
+ requires_grad=False)
121
+ layer.register_parameter("w13_input_scale", w13_input_scale)
122
+ set_weight_attrs(w13_input_scale, extra_weight_attrs)
123
+
124
+ w2_input_scale = torch.nn.Parameter(torch.ones(
125
+ num_experts, dtype=torch.float32),
126
+ requires_grad=False)
127
+ layer.register_parameter("w2_input_scale", w2_input_scale)
128
+ set_weight_attrs(w2_input_scale, extra_weight_attrs)
129
+ else:
130
+ layer.w13_input_scale = None
131
+ layer.w2_input_scale = None
132
+
133
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
134
+ # Fp8 moe kernels require a single activation scale.
135
+ # We take the max of all the scales in case they differ.
136
+ if self.static_input_scales:
137
+ if (layer.w13_input_scale is None or layer.w2_input_scale is None):
138
+ raise ValueError(
139
+ "QuantConfig has static quantization, but found "
140
+ "activation scales are None.")
141
+ if (not all_close_1d(layer.w13_input_scale)
142
+ or not all_close_1d(layer.w2_input_scale)):
143
+ print_warning_once(
144
+ "Found input_scales that are not equal for "
145
+ "fp8 MoE layer. Using the maximum across experts "
146
+ "for each layer. ")
147
+ layer.w13_input_scale = torch.nn.Parameter(
148
+ layer.w13_input_scale.max(), requires_grad=False)
149
+ layer.w2_input_scale = torch.nn.Parameter(
150
+ layer.w2_input_scale.max(), requires_grad=False)
151
+
152
+ # If rocm, normalize the weights and scales to e4m3fnuz
153
+ if is_hip():
154
+ # Normalize the weights and scales
155
+ w13_weight, w13_weight_scale, w13_input_scale = \
156
+ normalize_e4m3fn_to_e4m3fnuz(
157
+ layer.w13_weight, layer.w13_weight_scale,
158
+ layer.w13_input_scale)
159
+ w2_weight, w2_weight_scale, w2_input_scale = \
160
+ normalize_e4m3fn_to_e4m3fnuz(
161
+ layer.w2_weight, layer.w2_weight_scale,
162
+ layer.w2_input_scale)
163
+ # Reset the parameter
164
+ layer.w13_weight = torch.nn.Parameter(w13_weight,
165
+ requires_grad=False)
166
+ layer.w13_weight_scale = torch.nn.Parameter(w13_weight_scale,
167
+ requires_grad=False)
168
+ if w13_input_scale is not None:
169
+ layer.w13_input_scale = torch.nn.Parameter(w13_input_scale,
170
+ requires_grad=False)
171
+ layer.w2_weight = torch.nn.Parameter(w2_weight,
172
+ requires_grad=False)
173
+ layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale,
174
+ requires_grad=False)
175
+ if w2_input_scale is not None:
176
+ layer.w2_input_scale = torch.nn.Parameter(w2_input_scale,
177
+ requires_grad=False)
178
+
179
+ # Fp8 moe kernel needs single weight scale for w13 per expert.
180
+ # We take the max then dequant and requant each expert.
181
+ assert layer.w13_weight_scale is not None
182
+ shard_size = layer.intermediate_size_per_partition
183
+ max_w13_scales = layer.w13_weight_scale.max(dim=1).values
184
+ for expert_id in range(layer.num_experts):
185
+ start = 0
186
+ for shard_id in range(2):
187
+ dq_weight = per_tensor_dequantize(
188
+ layer.w13_weight[expert_id][start:start + shard_size, :],
189
+ layer.w13_weight_scale[expert_id][shard_id])
190
+ layer.w13_weight[expert_id][
191
+ start:start + shard_size, :], _ = ops.scaled_fp8_quant(
192
+ dq_weight, max_w13_scales[expert_id])
193
+ start += shard_size
194
+
195
+ layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales,
196
+ requires_grad=False)
197
+
198
+ def apply(
199
+ self,
200
+ layer: torch.nn.Module,
201
+ x: torch.Tensor,
202
+ router_logits: torch.Tensor,
203
+ top_k: int,
204
+ renormalize: bool = True,
205
+ use_grouped_topk: bool = False,
206
+ num_expert_group: Optional[int] = None,
207
+ topk_group: Optional[int] = None,
208
+ custom_routing_function: Optional[Callable] = None,
209
+ ) -> torch.Tensor:
210
+
211
+ from vllm.model_executor.layers.fused_moe import fused_experts
212
+
213
+ topk_weights, topk_ids = FusedMoE.select_experts(
214
+ hidden_states=x,
215
+ router_logits=router_logits,
216
+ use_grouped_topk=use_grouped_topk,
217
+ top_k=top_k,
218
+ renormalize=renormalize,
219
+ topk_group=topk_group,
220
+ num_expert_group=num_expert_group,
221
+ custom_routing_function=custom_routing_function)
222
+
223
+ return fused_experts(x,
224
+ layer.w13_weight,
225
+ layer.w2_weight,
226
+ topk_weights=topk_weights,
227
+ topk_ids=topk_ids,
228
+ inplace=True,
229
+ use_fp8_w8a8=True,
230
+ w1_scale=layer.w13_weight_scale,
231
+ w2_scale=layer.w2_weight_scale,
232
+ a1_scale=layer.w13_input_scale,
233
+ a2_scale=layer.w2_input_scale)
234
+
235
+
236
+class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
237
+
238
def __init__(
239
self,
240
quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
vllm/model_executor/models/phimoe.py
+2
-2
@@ -321,13 +321,13 @@ class PhiMoEAttention(nn.Module):
321
self.total_num_heads,
322
self.total_num_kv_heads,
323
bias=True,
324
- quant_config=None,
324
+ quant_config=quant_config,
325
)
326
self.o_proj = RowParallelLinear(
327
self.total_num_heads * self.head_dim,
328
hidden_size,
329
bias=True,
330
- quant_config=None,
330
+ quant_config=quant_config,
331
)
332
self.rotary_emb = get_rope(
333
self.head_dim,