A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Optimize FP8 support for MoE kernel / Mixtral via static scales (#4343)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Philipp Moritz committed
Apr 26, 2024 at 21:49 UTC
12628d3c787efd3483aaa74b5ae4175b28fd5805
7 files changed
+95
-18
csrc/ops.h
+6
-1
@@ -146,7 +146,12 @@ void gptq_shuffle(
146
torch::Tensor q_perm,
147
int bit);
148
149
-void scaled_fp8_quant(
149
+void static_scaled_fp8_quant(
150
+ torch::Tensor& out,
151
+ torch::Tensor& input,
152
+ torch::Tensor& scale);
153
+
154
+void dynamic_scaled_fp8_quant(
155
torch::Tensor& out,
156
torch::Tensor& input,
157
torch::Tensor& scale);
csrc/pybind.cpp
+2
-1
@@ -73,7 +73,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
73
ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
74
ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
75
ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
76
- ops.def("scaled_fp8_quant", &scaled_fp8_quant, "Compute FP8 quantized tensor and scaling factor");
76
+ ops.def("static_scaled_fp8_quant", &static_scaled_fp8_quant, "Compute FP8 quantized tensor for given scaling factor");
77
+ ops.def("dynamic_scaled_fp8_quant", &dynamic_scaled_fp8_quant, "Compute FP8 quantized tensor and scaling factor");
78
ops.def(
79
"moe_align_block_size",
80
&moe_align_block_size,
csrc/quantization/fp8/fp8_cuda_kernels.cu
+24
-1
@@ -74,7 +74,30 @@ __global__ void scaled_fp8_quant_kernel(
74
75
} // namespace vllm
76
77
-void scaled_fp8_quant(
77
+void static_scaled_fp8_quant(
78
+ torch::Tensor& out, // [..., d]
79
+ torch::Tensor& input, // [..., d]
80
+ torch::Tensor& scale) // [1]
81
+{
82
+ int64_t num_tokens = input.numel() / input.size(-1);
83
+ int64_t num_elems = input.numel();
84
+ dim3 grid(num_tokens);
85
+ dim3 block(1024);
86
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
87
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
88
+ VLLM_DISPATCH_FLOATING_TYPES(
89
+ input.scalar_type(),
90
+ "scaled_fp8_quant_kernel",
91
+ [&] {
92
+ vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
93
+ out.data_ptr<c10::Float8_e4m3fn>(),
94
+ input.data_ptr<scalar_t>(),
95
+ scale.data_ptr<float>(),
96
+ num_elems);
97
+ });
98
+}
99
+
100
+void dynamic_scaled_fp8_quant(
101
torch::Tensor& out, // [..., d]
102
torch::Tensor& input, // [..., d]
103
torch::Tensor& scale) // [1]
vllm/_custom_ops.py
+9
-3
@@ -168,10 +168,16 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
168
169
170
# fp8
171
-def scaled_fp8_quant(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
172
- scale = torch.zeros(1, device=input.device, dtype=torch.float32)
171
+def scaled_fp8_quant(
172
+ input: torch.Tensor,
173
+ scale: Optional[torch.Tensor] = None,
174
+) -> Tuple[torch.Tensor, torch.Tensor]:
175
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
174
- vllm_ops.scaled_fp8_quant(output, input, scale)
176
+ if scale is None:
177
+ scale = torch.zeros(1, device=input.device, dtype=torch.float32)
178
+ vllm_ops.dynamic_scaled_fp8_quant(output, input, scale)
179
+ else:
180
+ vllm_ops.static_scaled_fp8_quant(output, input, scale)
181
return output, scale
182
183
vllm/model_executor/layers/fused_moe/fused_moe.py
+9
-4
@@ -220,8 +220,9 @@ def moe_align_block_size(
220
221
222
def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
223
- B_scale: torch.Tensor, topk_weights: torch.Tensor,
224
- topk_ids: torch.Tensor,
223
+ A_scale: Optional[torch.Tensor],
224
+ B_scale: Optional[torch.Tensor],
225
+ topk_weights: torch.Tensor, topk_ids: torch.Tensor,
226
sorted_token_ids: torch.Tensor,
227
expert_ids: torch.Tensor,
228
num_tokens_post_padded: torch.Tensor,
@@ -232,10 +233,10 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
233
assert sorted_token_ids.stride(0) == 1
234
235
if not use_fp8:
235
- A_scale = None
236
+ assert A_scale is None
237
assert B_scale is None
238
else:
238
- A, A_scale = ops.scaled_fp8_quant(A)
239
+ A, A_scale = ops.scaled_fp8_quant(A, A_scale)
240
assert B_scale is not None
241
242
grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
@@ -318,6 +319,8 @@ def fused_moe(
319
use_fp8: bool = False,
320
w1_scale: Optional[torch.Tensor] = None,
321
w2_scale: Optional[torch.Tensor] = None,
322
+ a1_scale: Optional[torch.Tensor] = None,
323
+ a2_scale: Optional[torch.Tensor] = None,
324
) -> torch.Tensor:
325
"""
326
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -434,6 +437,7 @@ def fused_moe(
437
invoke_fused_moe_kernel(hidden_states,
438
w1,
439
intermediate_cache1,
440
+ a1_scale,
441
w1_scale,
442
topk_weights,
443
topk_ids,
@@ -451,6 +455,7 @@ def fused_moe(
455
invoke_fused_moe_kernel(intermediate_cache2,
456
w2,
457
intermediate_cache3,
458
+ a2_scale,
459
w2_scale,
460
topk_weights,
461
topk_ids,
vllm/model_executor/layers/quantization/fp8.py
+8
-1
@@ -14,6 +14,12 @@ from vllm.model_executor.utils import set_weight_attrs
14
class Fp8Config(QuantizationConfig):
15
"""Config class for FP8."""
16
17
+ def __init__(
18
+ self,
19
+ activation_scheme: str = "dynamic",
20
+ ) -> None:
21
+ self.activation_scheme = activation_scheme
22
+
23
@classmethod
24
def get_name(cls) -> str:
25
return "fp8"
@@ -35,7 +41,8 @@ class Fp8Config(QuantizationConfig):
41
42
@classmethod
43
def from_config(cls, config: Dict[str, Any]) -> "Fp8Config":
38
- return cls()
44
+ activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
45
+ return cls(activation_scheme)
46
47
def get_quant_method(
48
self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
vllm/model_executor/models/mixtral.py
+37
-7
@@ -105,6 +105,13 @@ class MixtralMoE(nn.Module):
105
device="cuda",
106
dtype=self.params_dtype))
107
108
+ set_weight_attrs(self.ws, {
109
+ "weight_loader": self.weight_loader,
110
+ })
111
+ set_weight_attrs(self.w2s, {
112
+ "weight_loader": self.weight_loader,
113
+ })
114
+
115
# Scaling factors for FP8 weights
116
self.ws_scale = nn.Parameter(
117
torch.ones(
@@ -115,12 +122,23 @@ class MixtralMoE(nn.Module):
122
self.num_total_experts, device="cuda", dtype=torch.float32),
123
requires_grad=False) if self.use_fp8 else None
124
118
- set_weight_attrs(self.ws, {
119
- "weight_loader": self.weight_loader,
120
- })
121
- set_weight_attrs(self.w2s, {
122
- "weight_loader": self.weight_loader,
123
- })
125
+ # Scaling factors for FP8 activations
126
+ need_act_scales = (self.use_fp8
127
+ and quant_config.activation_scheme == "static")
128
+ self.as_scale = nn.Parameter(
129
+ torch.zeros(1, device="cuda", dtype=torch.float32),
130
+ requires_grad=False) if need_act_scales else None
131
+ self.a2s_scale = nn.Parameter(
132
+ torch.zeros(1, device="cuda", dtype=torch.float32),
133
+ requires_grad=False) if need_act_scales else None
134
+
135
+ if need_act_scales:
136
+ set_weight_attrs(self.as_scale, {
137
+ "weight_loader": self.weight_loader,
138
+ })
139
+ set_weight_attrs(self.a2s_scale, {
140
+ "weight_loader": self.weight_loader,
141
+ })
142
143
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
144
weight_name: str, expert_id: int):
@@ -135,6 +153,8 @@ class MixtralMoE(nn.Module):
153
shard_size:2 * shard_size, :] = loaded_weight[shard, :]
154
if weight_name.endswith("w2.weight"):
155
param_data[expert_id, :, :] = loaded_weight[:, shard]
156
+ if "act_scale" in weight_name:
157
+ param_data[:] = param_data[:].max(loaded_weight)
158
159
def process_weights_after_loading(self):
160
if self.use_fp8:
@@ -162,7 +182,9 @@ class MixtralMoE(nn.Module):
182
inplace=True,
183
use_fp8=self.use_fp8,
184
w1_scale=self.ws_scale,
165
- w2_scale=self.w2s_scale)
185
+ w2_scale=self.w2s_scale,
186
+ a1_scale=self.as_scale,
187
+ a2_scale=self.a2s_scale)
188
189
if self.tp_size > 1:
190
final_hidden_states = tensor_model_parallel_all_reduce(
@@ -443,11 +465,19 @@ class MixtralForCausalLM(nn.Module):
465
]
466
467
expert_params_mapping = [
468
+ # These are the weights for the experts
469
# (param_name, weight_name, expert_id)
470
("ws" if weight_name in ["w1", "w3"] else "w2s",
471
f"experts.{expert_id}.{weight_name}.weight", expert_id)
472
for expert_id in range(self.config.num_local_experts)
473
for weight_name in ["w1", "w2", "w3"]
474
+ ] + [
475
+ # These are the activation scales for the experts
476
+ # (param_name, weight_name, expert_id)
477
+ ("as_scale" if weight_name in ["w1", "w3"] else "a2s_scale",
478
+ f"experts.{expert_id}.{weight_name}.act_scale", expert_id)
479
+ for expert_id in range(self.config.num_local_experts)
480
+ for weight_name in ["w1", "w2", "w3"]
481
]
482
483
params_dict = dict(self.named_parameters())