@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eace8bf0b9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Kernel] FP8 support for MoE kernel / Mixtral (#4244)

This PR is the first step towards fixing https://github.com/vllm-project/vllm/pull/3208 It implements dynamic per-tensor scaling (see https://github.com/vllm-project/vllm/pull/4118), so users do not need to compute activation scales on a calibration dataset and they also don't need to convert their model checkpoints. It is enough to specify the `quantization="fp8"` argument. You can try out the PR like this: ```python from vllm import LLM, SamplingParams prompts = [ "Hello, my name is", "The president of the United States is", "The capital of France is", "The future of AI is", ] sampling_params = SamplingParams(temperature=0.8, top_p=0.95) llm = LLM(model="mistralai/Mixtral-8x7B-Instruct-v0.1", tensor_parallel_size=2, quantization="fp8") outputs = llm.generate(prompts, sampling_params) # Print the outputs. for output in outputs: prompt = output.prompt generated_text = output.outputs[0].text print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") ``` **Performance**: For this PR, the focus is on making the code clean (while still trying to get reasonable performance), there is a bunch of optimizations that we will submit as a follow up PR that significantly improve the performance (similar to the numbers in https://github.com/vllm-project/vllm/pull/3954). With this PR, the results are as follows: <img width="725" alt="Screenshot 2024-04-21 at 1 31 50 PM" src="https://github.com/vllm-project/vllm/assets/113316/d8fe1118-07a0-4d4e-8530-37a77d465a03"> **Accuracy**: The accuracy with this PR on MMLU on `mistralai/Mixtral-8x7B-v0.1` is as follows: ``` | Groups |Version|Filter|n-shot|Metric|Value | |Stderr| |------------------|-------|------|-----:|------|-----:|---|-----:| |mmlu |N/A |none | 0|acc |0.7018|± |0.0036| | - humanities |N/A |none | 5|acc |0.6472|± |0.0065| | - other |N/A |none | 5|acc |0.7673|± |0.0072| | - social_sciences|N/A |none | 5|acc |0.8099|± |0.0070| | - stem |N/A |none | 5|acc |0.6131|± |0.0083| ``` this compares favorably with the fp16 results which are ``` | Groups |Version|Filter|n-shot|Metric|Value | |Stderr| |------------------|-------|------|-----:|------|-----:|---|-----:| |mmlu |N/A |none | 0|acc |0.7020|± |0.1313| | - humanities |N/A |none | 5|acc |0.6425|± |0.1349| | - other |N/A |none | 5|acc |0.7744|± |0.1038| | - social_sciences|N/A |none | 5|acc |0.8131|± |0.0695| | - stem |N/A |none | 5|acc |0.6108|± |0.1383| ``` Happy hacking!

Philipp Moritz committed Apr 23, 2024 at 18:18 UTC eace8bf0b9118877c390e6d490502214c39db132
10 files changed +385 -21
CMakeLists.txt
+1
@@ -167,6 +167,7 @@ set(VLLM_EXT_SRC
167 "csrc/layernorm_kernels.cu"
168 "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
169 "csrc/quantization/gptq/q_gemm.cu"
170 + "csrc/quantization/fp8/fp8_cuda_kernels.cu"
171 "csrc/cuda_utils_kernels.cu"
172 "csrc/moe_align_block_size_kernels.cu"
173 "csrc/pybind.cpp")
csrc/ops.h
+5
@@ -146,6 +146,11 @@ void gptq_shuffle(
146 torch::Tensor q_perm,
147 int bit);
148
149 +void scaled_fp8_quant(
150 + torch::Tensor& out,
151 + torch::Tensor& input,
152 + torch::Tensor& scale);
153 +
154 void moe_align_block_size(
155 torch::Tensor topk_ids,
156 int num_experts,
csrc/pybind.cpp
+1
@@ -73,6 +73,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
73 ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
74 ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
75 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
76 + ops.def("scaled_fp8_quant", &scaled_fp8_quant, "Compute FP8 quantized tensor and scaling factor");
77 ops.def(
78 "moe_align_block_size",
79 &moe_align_block_size,
csrc/quantization/fp8/fp8_cuda_kernels.cu new
+103
@@ -0,0 +1,103 @@
1 +#include <ATen/cuda/CUDAContext.h>
2 +#include <torch/extension.h>
3 +#include <c10/cuda/CUDAGuard.h>
4 +
5 +#include <cmath>
6 +
7 +#include "cuda_compat.h"
8 +#include "dispatch_utils.h"
9 +
10 +namespace vllm {
11 +
12 +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
13 + float old;
14 + old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) :
15 + __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value)));
16 +
17 + return old;
18 +}
19 +
20 +// Compute the absolute maximum m of the input tensor and store
21 +// m / float8_e4m3::max() in *scale. Each thread block performs a
22 +// reduction tree and the memory in scale is atomically updated.
23 +// So to get the right answer, *scale needs to be initialized to
24 +// a value <= 0.0 and we need to wait for all thread blocks to
25 +// finish before consuming *scale.
26 +template<typename scalar_t>
27 +__global__ void segmented_max_reduction(
28 + float* __restrict__ scale,
29 + const scalar_t* __restrict__ input,
30 + int64_t num_elems) {
31 + __shared__ float cache[1024];
32 + int i = blockDim.x * blockIdx.x + threadIdx.x;
33 +
34 + // First store maximum for all values processes by
35 + // the current thread in cache[threadIdx.x]
36 + scalar_t tmp = 0.0;
37 + while (i < num_elems) {
38 + float x = static_cast<float>(input[i]);
39 + tmp = max(tmp, fabs(x));
40 + i += blockDim.x * gridDim.x;
41 + }
42 + cache[threadIdx.x] = tmp;
43 +
44 + __syncthreads();
45 +
46 + // Now perform parallel reduction within the thread block
47 + int ib = blockDim.x / 2;
48 + while (ib != 0) {
49 + if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
50 + cache[threadIdx.x] = cache[threadIdx.x + ib];
51 + }
52 + __syncthreads();
53 + ib /= 2;
54 + }
55 + // Finally, since cache[0] contains the maximum for this thread block,
56 + // atomically write the max to the target location
57 + if (threadIdx.x == 0) {
58 + atomicMaxFloat(scale, cache[0] / std::numeric_limits<c10::Float8_e4m3fn>::max());
59 + }
60 +}
61 +
62 +template<typename scalar_t>
63 +__global__ void scaled_fp8_quant_kernel(
64 + c10::Float8_e4m3fn* __restrict__ out,
65 + const scalar_t* __restrict__ input,
66 + const float* __restrict__ scale,
67 + int64_t num_elems) {
68 + int i = blockDim.x * blockIdx.x + threadIdx.x;
69 + while (i < num_elems) {
70 + out[i] = static_cast<c10::Float8_e4m3fn>(input[i] / *scale);
71 + i += blockDim.x * gridDim.x;
72 + }
73 +}
74 +
75 +} // namespace vllm
76 +
77 +void scaled_fp8_quant(
78 + torch::Tensor& out, // [..., d]
79 + torch::Tensor& input, // [..., d]
80 + torch::Tensor& scale) // [1]
81 +{
82 + int64_t num_tokens = input.numel() / input.size(-1);
83 + int64_t num_elems = input.numel();
84 + dim3 grid(num_tokens);
85 + dim3 block(1024);
86 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
87 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
88 + VLLM_DISPATCH_FLOATING_TYPES(
89 + input.scalar_type(),
90 + "scaled_fp8_quant_kernel",
91 + [&] {
92 + vllm::segmented_max_reduction<scalar_t><<<grid, block, 0, stream>>>(
93 + scale.data_ptr<float>(),
94 + input.data_ptr<scalar_t>(),
95 + num_elems);
96 + vllm::scaled_fp8_quant_kernel<scalar_t><<<grid, block, 0, stream>>>(
97 + out.data_ptr<c10::Float8_e4m3fn>(),
98 + input.data_ptr<scalar_t>(),
99 + scale.data_ptr<float>(),
100 + num_elems);
101 + });
102 +}
103 +
vllm/_custom_ops.py
+9 -1
@@ -1,4 +1,4 @@
1 -from typing import Dict, Optional
1 +from typing import Dict, Optional, Tuple
2
3 import torch
4
@@ -153,6 +153,14 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
153 size_n, size_k)
154
155
156 +# fp8
157 +def scaled_fp8_quant(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
158 + scale = torch.zeros(1, device=input.device, dtype=torch.float32)
159 + output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
160 + vllm_ops.scaled_fp8_quant(output, input, scale)
161 + return output, scale
162 +
163 +
164 # moe
165 def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
166 block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 128,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 8,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 256,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 8,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 8,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 8,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/fused_moe.py
+75 -18
@@ -21,6 +21,8 @@ def fused_moe_kernel(
21 a_ptr,
22 b_ptr,
23 c_ptr,
24 + a_scale_ptr,
25 + b_scale_ptr,
26 topk_weights_ptr,
27 sorted_token_ids_ptr,
28 expert_ids_ptr,
@@ -49,6 +51,7 @@ def fused_moe_kernel(
51 MUL_ROUTED_WEIGHT: tl.constexpr,
52 top_k: tl.constexpr,
53 compute_type: tl.constexpr,
54 + use_fp8: tl.constexpr,
55 ):
56 """
57 Implements the fused computation for a Mixture of Experts (MOE) using
@@ -111,6 +114,10 @@ def fused_moe_kernel(
114 b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
115 offs_bn[None, :] * stride_bn)
116
117 + if use_fp8:
118 + a_scale = tl.load(a_scale_ptr)
119 + b_scale = tl.load(b_scale_ptr + off_experts)
120 +
121 # -----------------------------------------------------------
122 # Iterate to compute a block of the C matrix.
123 # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
@@ -129,7 +136,10 @@ def fused_moe_kernel(
136 mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
137 other=0.0)
138 # We accumulate along the K dimension.
132 - accumulator += tl.dot(a, b)
139 + if use_fp8:
140 + accumulator = tl.dot(a, b, acc=accumulator)
141 + else:
142 + accumulator += tl.dot(a, b)
143 # Advance the ptrs to the next K block.
144 a_ptrs += BLOCK_SIZE_K * stride_ak
145 b_ptrs += BLOCK_SIZE_K * stride_bk
@@ -140,7 +150,10 @@ def fused_moe_kernel(
150 other=0)
151 accumulator = accumulator * moe_weight[:, None]
152
143 - accumulator = accumulator.to(compute_type)
153 + if use_fp8:
154 + accumulator = (accumulator * a_scale * b_scale).to(compute_type)
155 + else:
156 + accumulator = accumulator.to(compute_type)
157 # -----------------------------------------------------------
158 # Write back the block of the output
159 offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
@@ -207,15 +220,24 @@ def moe_align_block_size(
220
221
222 def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
210 - topk_weights: torch.Tensor, topk_ids: torch.Tensor,
223 + B_scale: torch.Tensor, topk_weights: torch.Tensor,
224 + topk_ids: torch.Tensor,
225 sorted_token_ids: torch.Tensor,
226 expert_ids: torch.Tensor,
227 num_tokens_post_padded: torch.Tensor,
228 mul_routed_weight: bool, top_k: int,
215 - config: Dict[str, Any]) -> None:
229 + config: Dict[str, Any], compute_type: tl.dtype,
230 + use_fp8: bool) -> None:
231 assert topk_weights.stride(1) == 1
232 assert sorted_token_ids.stride(0) == 1
233
234 + if not use_fp8:
235 + A_scale = None
236 + assert B_scale is None
237 + else:
238 + A, A_scale = ops.scaled_fp8_quant(A)
239 + assert B_scale is not None
240 +
241 grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
242 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
243
@@ -223,6 +245,8 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
245 A,
246 B,
247 C,
248 + A_scale,
249 + B_scale,
250 topk_weights,
251 sorted_token_ids,
252 expert_ids,
@@ -240,18 +264,21 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
264 C.stride(2),
265 MUL_ROUTED_WEIGHT=mul_routed_weight,
266 top_k=top_k,
243 - compute_type=tl.bfloat16 if A.dtype == torch.bfloat16 else tl.float16,
267 + compute_type=compute_type,
268 + use_fp8=use_fp8,
269 **config,
270 )
271
272
248 -def get_config_file_name(E: int, N: int) -> str:
273 +def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
274 device_name = torch.cuda.get_device_name().replace(" ", "_")
250 - return f"E={E},N={N},device_name={device_name}.json"
275 + dtype_selector = "" if not dtype else f",dtype={dtype}"
276 + return f"E={E},N={N},device_name={device_name}{dtype_selector}.json"
277
278
279 @functools.lru_cache
254 -def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
280 +def get_moe_configs(E: int, N: int,
281 + dtype: Optional[str]) -> Optional[Dict[int, Any]]:
282 """
283 Return optimized configurations for the fused MoE kernel.
284
@@ -263,7 +290,7 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
290
291 # First look up if an optimized configuration is available in the configs
292 # directory
266 - json_file_name = get_config_file_name(E, N)
293 + json_file_name = get_config_file_name(E, N, dtype)
294
295 config_file_path = os.path.join(
296 os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
@@ -288,6 +315,9 @@ def fused_moe(
315 renormalize: bool,
316 inplace: bool = False,
317 override_config: Optional[Dict[str, Any]] = None,
318 + use_fp8: bool = False,
319 + w1_scale: Optional[torch.Tensor] = None,
320 + w2_scale: Optional[torch.Tensor] = None,
321 ) -> torch.Tensor:
322 """
323 This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -305,6 +335,12 @@ def fused_moe(
335 Defaults to False.
336 - override_config (Optional[Dict[str, Any]]): Optional override
337 for the kernel configuration.
338 + - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
339 + products for w1 and w2. Defaults to False.
340 + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
341 + w1.
342 + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
343 + w2.
344
345 Returns:
346 - torch.Tensor: The output tensor after applying the MoE layer.
@@ -358,7 +394,8 @@ def fused_moe(
394 config = override_config
395 else:
396 # First try to load optimal config from the file
361 - configs = get_moe_configs(E, w2.shape[2])
397 + configs = get_moe_configs(E, w2.shape[2],
398 + "float8" if use_fp8 else None)
399
400 if configs:
401 # If an optimal configuration map has been found, look up the
@@ -394,17 +431,37 @@ def fused_moe(
431 sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
432 topk_ids, config['BLOCK_SIZE_M'], E)
433
397 - invoke_fused_moe_kernel(hidden_states, w1, intermediate_cache1,
398 - topk_weights, topk_ids, sorted_token_ids,
399 - expert_ids, num_tokens_post_padded, False,
400 - topk_ids.shape[1], config)
434 + invoke_fused_moe_kernel(hidden_states,
435 + w1,
436 + intermediate_cache1,
437 + w1_scale,
438 + topk_weights,
439 + topk_ids,
440 + sorted_token_ids,
441 + expert_ids,
442 + num_tokens_post_padded,
443 + False,
444 + topk_ids.shape[1],
445 + config,
446 + compute_type=tl.float16,
447 + use_fp8=use_fp8)
448
449 ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
450
404 - invoke_fused_moe_kernel(intermediate_cache2, w2, intermediate_cache3,
405 - topk_weights, topk_ids, sorted_token_ids,
406 - expert_ids, num_tokens_post_padded, True, 1,
407 - config)
451 + invoke_fused_moe_kernel(intermediate_cache2,
452 + w2,
453 + intermediate_cache3,
454 + w2_scale,
455 + topk_weights,
456 + topk_ids,
457 + sorted_token_ids,
458 + expert_ids,
459 + num_tokens_post_padded,
460 + True,
461 + 1,
462 + config,
463 + compute_type=tl.float16,
464 + use_fp8=use_fp8)
465
466 if inplace:
467 return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
vllm/model_executor/model_loader/loader.py
+2
@@ -232,6 +232,8 @@ class DefaultModelLoader(BaseModelLoader):
232 linear_method = getattr(module, "linear_method", None)
233 if linear_method is not None:
234 linear_method.process_weights_after_loading(module)
235 + if hasattr(module, "process_weights_after_loading"):
236 + module.process_weights_after_loading()
237 return model.eval()
238
239
vllm/model_executor/model_loader/utils.py
+1
@@ -24,6 +24,7 @@ def get_model_architecture(
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 if (model_config.quantization is not None
27 + and model_config.quantization != "fp8"
28 and "MixtralForCausalLM" in architectures):
29 architectures = ["QuantMixtralForCausalLM"]
30
vllm/model_executor/models/mixtral.py
+42 -2
@@ -39,6 +39,8 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
39 ReplicatedLinear,
40 RowParallelLinear)
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 +from vllm.model_executor.layers.quantization.fp8 import (Fp8LinearMethod,
43 + per_tensor_quantize)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 from vllm.model_executor.layers.sampler import Sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -47,6 +49,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.model_executor.utils import set_weight_attrs
51 from vllm.sequence import SamplerOutput
52 +from vllm.utils import print_warning_once
53
54
55 class MixtralMoE(nn.Module):
@@ -66,6 +69,7 @@ class MixtralMoE(nn.Module):
69 intermediate_size: int,
70 params_dtype: Optional[torch.dtype] = None,
71 tp_size: Optional[int] = None,
72 + linear_method: Optional[LinearMethodBase] = None,
73 ):
74 super().__init__()
75 self.tp_size = tp_size or get_tensor_model_parallel_world_size()
@@ -73,6 +77,9 @@ class MixtralMoE(nn.Module):
77 self.top_k = top_k
78 self.hidden_size = hidden_size
79 self.intermediate_size = intermediate_size // self.tp_size
80 + # FIXME(pcmoritz): Make this more general to support different
81 + # quantization schemes
82 + self.use_fp8 = isinstance(linear_method, Fp8LinearMethod)
83
84 if params_dtype is None:
85 params_dtype = torch.get_default_dtype()
@@ -97,6 +104,16 @@ class MixtralMoE(nn.Module):
104 device="cuda",
105 dtype=self.params_dtype))
106
107 + # Scaling factors for FP8 weights
108 + self.ws_scale = nn.Parameter(
109 + torch.ones(
110 + self.num_total_experts, device="cuda", dtype=torch.float32),
111 + requires_grad=False) if self.use_fp8 else None
112 + self.w2s_scale = nn.Parameter(
113 + torch.ones(
114 + self.num_total_experts, device="cuda", dtype=torch.float32),
115 + requires_grad=False) if self.use_fp8 else None
116 +
117 set_weight_attrs(self.ws, {
118 "weight_loader": self.weight_loader,
119 })
@@ -118,6 +135,18 @@ class MixtralMoE(nn.Module):
135 if weight_name.endswith("w2.weight"):
136 param_data[expert_id, :, :] = loaded_weight[:, shard]
137
138 + def process_weights_after_loading(self):
139 + if self.use_fp8:
140 + ws = torch.empty_like(self.ws.data, dtype=torch.float8_e4m3fn)
141 + w2s = torch.empty_like(self.w2s.data, dtype=torch.float8_e4m3fn)
142 + for expert in range(self.num_total_experts):
143 + ws[expert, :, :], self.ws_scale[expert] = per_tensor_quantize(
144 + self.ws.data[expert, :, :])
145 + w2s[expert, :, :], self.w2s_scale[
146 + expert] = per_tensor_quantize(self.w2s.data[expert, :, :])
147 + self.ws = nn.Parameter(ws, requires_grad=False)
148 + self.w2s = nn.Parameter(w2s, requires_grad=False)
149 +
150 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
151 num_tokens, hidden_size = hidden_states.shape
152 hidden_states = hidden_states.view(-1, self.hidden_size)
@@ -129,7 +158,10 @@ class MixtralMoE(nn.Module):
158 router_logits,
159 self.top_k,
160 renormalize=True,
132 - inplace=True)
161 + inplace=True,
162 + use_fp8=self.use_fp8,
163 + w1_scale=self.ws_scale,
164 + w2_scale=self.w2s_scale)
165
166 if self.tp_size > 1:
167 final_hidden_states = tensor_model_parallel_all_reduce(
@@ -171,6 +203,13 @@ class MixtralAttention(nn.Module):
203 self.rope_theta = rope_theta
204 self.sliding_window = sliding_window
205
206 + if isinstance(linear_method, Fp8LinearMethod):
207 + print_warning_once(
208 + "For Mixtral FP8 quantization, we currently do not quantize "
209 + "the attention layers until their FP8 performance is improved."
210 + )
211 + linear_method = None
212 +
213 self.qkv_proj = QKVParallelLinear(
214 hidden_size,
215 self.head_dim,
@@ -238,7 +277,8 @@ class MixtralDecoderLayer(nn.Module):
277 num_experts=config.num_local_experts,
278 top_k=config.num_experts_per_tok,
279 hidden_size=config.hidden_size,
241 - intermediate_size=config.intermediate_size)
280 + intermediate_size=config.intermediate_size,
281 + linear_method=linear_method)
282 self.input_layernorm = RMSNorm(config.hidden_size,
283 eps=config.rms_norm_eps)
284 self.post_attention_layernorm = RMSNorm(config.hidden_size,