@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6cd5e5b07e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Fused MoE Marlin support for GPTQ (#8217)

Dipika Sikka committed Sep 9, 2024 at 23:02 UTC 6cd5e5b07e4415d064d93b8a66331a097bd9287e
19 files changed +912 -204
.buildkite/test-pipeline.yaml
+12 -1
@@ -386,7 +386,18 @@ steps:
386 - vllm/
387 - tests/weight_loading
388 commands:
389 - - bash weight_loading/run_model_weight_loading_test.sh
389 + - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
390 +
391 +- label: Weight Loading Multiple GPU Test - Large Models # optional
392 + working_dir: "/vllm-workspace/tests"
393 + num_gpus: 2
394 + gpu: a100
395 + optional: true
396 + source_file_dependencies:
397 + - vllm/
398 + - tests/weight_loading
399 + commands:
400 + - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt
401
402
403 ##### multi gpus test #####
csrc/moe/marlin_moe_ops.cu
+1 -1
@@ -1737,4 +1737,4 @@ torch::Tensor marlin_gemm_moe(
1737 moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1738 thread_n, sms, max_par, replicate_input, apply_weights);
1739 return c;
1740 -}
\ No newline at end of file
1740 +}
csrc/moe/marlin_moe_ops.h
+1 -1
@@ -9,4 +9,4 @@ torch::Tensor marlin_gemm_moe(
9 const torch::Tensor& g_idx, const torch::Tensor& perm,
10 torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11 bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12 - bool replicate_input, bool apply_weights);
\ No newline at end of file
12 + bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
-1
@@ -16,7 +16,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
16 "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
17 "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
18 "bool replicate_input, bool apply_weights) -> Tensor");
19 -
19 m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
20 #endif
21 }
tests/kernels/test_moe.py
+217 -4
@@ -2,6 +2,8 @@
2
3 Run `pytest tests/kernels/test_moe.py`.
4 """
5 +from typing import List
6 +
7 import pytest
8 import torch
9 from transformers import MixtralConfig
@@ -9,7 +11,13 @@ from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
11
12 from vllm.model_executor.layers.activation import SiluAndMul
13 from vllm.model_executor.layers.fused_moe import fused_moe
14 +from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15 + fused_marlin_moe, single_marlin_moe)
16 +from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
17 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
18 + marlin_quantize)
19 from vllm.model_executor.models.mixtral import MixtralMoE
20 +from vllm.scalar_type import scalar_types
21
22
23 def torch_moe(a, w1, w2, score, topk):
@@ -29,6 +37,20 @@ def torch_moe(a, w1, w2, score, topk):
37 topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
38
39
40 +def torch_moe_single(a, w, score, topk):
41 + B, D = a.shape
42 + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
43 + out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
44 + score = torch.softmax(score, dim=-1, dtype=torch.float32)
45 + _, topk_ids = torch.topk(score, topk)
46 + topk_ids = topk_ids.view(-1)
47 + for i in range(w.shape[0]):
48 + mask = topk_ids == i
49 + if mask.sum():
50 + out[mask] = a[mask] @ w[i].transpose(0, 1)
51 + return (out.view(B, -1, w.shape[1])).sum(dim=1)
52 +
53 +
54 @pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
55 @pytest.mark.parametrize("n", [2048, 256, 1024])
56 @pytest.mark.parametrize("k", [128, 511, 1024])
@@ -43,11 +65,11 @@ def test_fused_moe(
65 topk: int,
66 dtype: torch.dtype,
67 ):
46 - a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
47 - w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
48 - w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
68 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
69 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
70 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
71
50 - score = torch.randn((m, e), device='cuda', dtype=dtype)
72 + score = torch.randn((m, e), device="cuda", dtype=dtype)
73 triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
74 torch_output = torch_moe(a, w1, w2, score, topk)
75 torch.testing.assert_close(triton_output, torch_output, atol=1e-2, rtol=0)
@@ -99,3 +121,194 @@ def test_mixtral_moe(dtype: torch.dtype):
121 vllm_states,
122 rtol=mixtral_moe_tol[dtype],
123 atol=mixtral_moe_tol[dtype])
124 +
125 +
126 +def stack_and_dev(tensors: List[torch.Tensor]):
127 + dev = tensors[0].device
128 + return torch.stack(tensors, dim=0).to(dev)
129 +
130 +
131 +def compute_max_diff(output, output_ref):
132 + return torch.mean(torch.abs(output - output_ref)) / torch.mean(
133 + torch.abs(output_ref))
134 +
135 +
136 +@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
137 +@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
138 +@pytest.mark.parametrize("k", [128, 1024, 512])
139 +@pytest.mark.parametrize("e", [4, 8, 64])
140 +@pytest.mark.parametrize("topk", [2, 6])
141 +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
142 +@pytest.mark.parametrize("act_order", [True, False])
143 +def test_fused_marlin_moe(
144 + m: int,
145 + n: int,
146 + k: int,
147 + e: int,
148 + topk: int,
149 + group_size: int,
150 + act_order: bool,
151 +):
152 + torch.manual_seed(7)
153 +
154 + if topk > e:
155 + return
156 +
157 + # Filter act_order
158 + if act_order:
159 + if group_size == -1:
160 + return
161 + if group_size in (k, n):
162 + return
163 +
164 + quant_type = scalar_types.uint4b8
165 + dtype = torch.float16
166 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
167 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
168 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
169 + for i in range(w2.shape[0]):
170 + w2[0] = torch.eye(k, n, device="cuda", dtype=dtype)
171 +
172 + w_ref1_l = []
173 + qweight1_l = []
174 + scales1_l = []
175 + g_idx1_l = []
176 + sort_indices1_l = []
177 +
178 + for i in range(w1.shape[0]):
179 + test_perm = torch.randperm(k)
180 + w_ref1, qweight1, scales1, g_idx1, sort_indices1, _ = marlin_quantize(
181 + w1[i].transpose(1, 0), quant_type, group_size, act_order,
182 + test_perm)
183 + w_ref1_l.append(w_ref1)
184 + qweight1_l.append(qweight1)
185 + scales1_l.append(scales1)
186 + g_idx1_l.append(g_idx1)
187 + sort_indices1_l.append(sort_indices1)
188 +
189 + w_ref1 = stack_and_dev(w_ref1_l)
190 + qweight1 = stack_and_dev(qweight1_l).contiguous()
191 + scales1 = stack_and_dev(scales1_l)
192 + g_idx1 = stack_and_dev(g_idx1_l)
193 + sort_indices1 = stack_and_dev(sort_indices1_l)
194 +
195 + w_ref2_l = []
196 + qweight2_l = []
197 + scales2_l = []
198 + g_idx2_l = []
199 + sort_indices2_l = []
200 +
201 + for i in range(w2.shape[0]):
202 + test_perm = torch.randperm(n)
203 + w_ref2, qweight2, scales2, g_idx2, sort_indices2, _ = marlin_quantize(
204 + w2[i].transpose(1, 0), quant_type, group_size, act_order,
205 + test_perm)
206 + w_ref2_l.append(w_ref2)
207 + qweight2_l.append(qweight2)
208 + scales2_l.append(scales2)
209 + g_idx2_l.append(g_idx2)
210 + sort_indices2_l.append(sort_indices2)
211 +
212 + w_ref2 = stack_and_dev(w_ref2_l)
213 + qweight2 = stack_and_dev(qweight2_l).contiguous()
214 + scales2 = stack_and_dev(scales2_l)
215 + g_idx2 = stack_and_dev(g_idx2_l)
216 + sort_indices2 = stack_and_dev(sort_indices2_l)
217 +
218 + score = torch.randn((m, e), device="cuda", dtype=dtype)
219 +
220 + topk_weights, topk_ids = fused_topk(a, score, topk, False)
221 +
222 + triton_output = fused_moe(
223 + a,
224 + w_ref1.transpose(1, 2).contiguous(),
225 + w_ref2.transpose(1, 2).contiguous(),
226 + score,
227 + topk,
228 + renormalize=False,
229 + )
230 + marlin_output = fused_marlin_moe(
231 + a,
232 + qweight1,
233 + qweight2,
234 + score,
235 + g_idx1,
236 + g_idx2,
237 + sort_indices1,
238 + sort_indices2,
239 + topk_weights,
240 + topk_ids,
241 + w1_scale=scales1,
242 + w2_scale=scales2,
243 + )
244 +
245 + assert compute_max_diff(marlin_output, triton_output) < 4e-2
246 +
247 +
248 +@pytest.mark.skip("This test is here for the sake of debugging, "
249 + "don't run it in automated tests.")
250 +@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
251 +@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
252 +@pytest.mark.parametrize("k", [128, 1024, 512])
253 +@pytest.mark.parametrize("e", [4, 8, 64])
254 +@pytest.mark.parametrize("topk", [2, 6])
255 +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
256 +@pytest.mark.parametrize("act_order", [True, False])
257 +def test_marlin_moe_mmm(
258 + m: int,
259 + n: int,
260 + k: int,
261 + e: int,
262 + topk: int,
263 + group_size: int,
264 + act_order: bool,
265 +):
266 + if topk > e:
267 + return
268 +
269 + # Filter act_order
270 + if act_order:
271 + if group_size == -1:
272 + return
273 + if group_size == k:
274 + return
275 +
276 + quant_type = scalar_types.uint4b8
277 + dtype = torch.float16
278 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
279 + w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
280 +
281 + w_ref_l = []
282 + qweights_l = []
283 + scales_l = []
284 + g_idx_l = []
285 + sort_indices_l = []
286 +
287 + for i in range(w.shape[0]):
288 + test_perm = torch.randperm(k)
289 + w_ref, qweight, scales, g_idx, sort_indices, _ = marlin_quantize(
290 + w[i].transpose(1, 0), quant_type, group_size, act_order, test_perm)
291 + w_ref_l.append(w_ref)
292 + qweights_l.append(qweight)
293 + scales_l.append(scales)
294 + g_idx_l.append(g_idx)
295 + sort_indices_l.append(sort_indices)
296 +
297 + w_ref = stack_and_dev(w_ref_l)
298 + qweight = stack_and_dev(qweights_l).contiguous()
299 + scales = stack_and_dev(scales_l)
300 + g_idx = stack_and_dev(g_idx_l)
301 + sort_indices = stack_and_dev(sort_indices_l)
302 +
303 + score = torch.randn((m, e), device="cuda", dtype=dtype)
304 + marlin_output = single_marlin_moe(a,
305 + qweight,
306 + scales,
307 + score,
308 + g_idx,
309 + sort_indices,
310 + topk,
311 + renormalize=False)
312 + torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
313 +
314 + assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/weight_loading/models-large.txt new
+3
@@ -0,0 +1,3 @@
1 +compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2 +compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3 +gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
\ No newline at end of file
tests/weight_loading/models.txt
-2
@@ -19,8 +19,6 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
19 compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
20 compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22 -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
23 -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
22 compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23 awq, casperhansen/mixtral-instruct-awq, main
24 awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/model_executor/layers/fused_moe/__init__.py
+10 -4
@@ -2,16 +2,22 @@ from vllm.model_executor.layers.fused_moe.layer import (
2 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
3 from vllm.triton_utils import HAS_TRITON
4
5 -__all__ = ["FusedMoE", "FusedMoEMethodBase", "FusedMoeWeightScaleSupported"]
5 +__all__ = [
6 + "FusedMoE",
7 + "FusedMoEMethodBase",
8 + "FusedMoeWeightScaleSupported",
9 +]
10
11 if HAS_TRITON:
8 -
12 + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
13 + fused_marlin_moe, single_marlin_moe)
14 from vllm.model_executor.layers.fused_moe.fused_moe import (
10 - fused_experts, fused_marlin_moe, fused_moe, fused_topk,
11 - get_config_file_name, grouped_topk)
15 + fused_experts, fused_moe, fused_topk, get_config_file_name,
16 + grouped_topk)
17
18 __all__ += [
19 "fused_marlin_moe",
20 + "single_marlin_moe",
21 "fused_moe",
22 "fused_topk",
23 "fused_experts",
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py new
+219
@@ -0,0 +1,219 @@
1 +"""Fused MoE utilities for GPTQ."""
2 +import functools
3 +from typing import Any, Dict, Optional
4 +
5 +import torch
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm.model_executor.layers.fused_moe.fused_moe import (
9 + fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10 +
11 +
12 +def single_marlin_moe(
13 + hidden_states: torch.Tensor,
14 + w: torch.Tensor,
15 + scales: torch.Tensor,
16 + gating_output: torch.Tensor,
17 + g_idx: torch.Tensor,
18 + perm: torch.Tensor,
19 + topk: int,
20 + renormalize: bool,
21 + override_config: Optional[Dict[str, Any]] = None) -> torch.Tensor:
22 + """
23 + This function computes the multiplication of hidden_states with expert
24 + weights used in Marlin MoE, using weights w and top-k gating mechanism.
25 + Its purpose is testing and debugging the fused MoE kernel.
26 +
27 + Parameters:
28 + - hidden_states (torch.Tensor): The input tensor to the Marlin Mul.
29 + - w (torch.Tensor): The set of expert weights.
30 + - scales (torch.Tensor): The quantization scales.
31 + - gating_output (torch.Tensor): The output of the gating operation
32 + (before softmax).
33 + - g_idx (torch.Tensor): The act_order indices.
34 + - perm (torch.Tensor): The act_order input permutation.
35 + - topk (int): The number of top-k experts to select.
36 + - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
37 + - override_config (Optional[Dict[str, Any]]): Optional override
38 + for the kernel configuration.
39 +
40 + Returns:
41 + - torch.Tensor: The output tensor after applying the MoE layer.
42 + """
43 + # Check constraints.
44 + assert hidden_states.shape[0] == gating_output.shape[0], (
45 + "Number of tokens mismatch")
46 + assert hidden_states.shape[1] == w.shape[1] * 16, "Hidden size mismatch"
47 + assert gating_output.shape[1] == w.shape[0], "Number of experts mismatch"
48 + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
49 + assert w.is_contiguous(), "Expert weights must be contiguous"
50 + assert hidden_states.dtype == torch.float16
51 +
52 + M, K = hidden_states.shape
53 + E = w.shape[0]
54 + N = w.shape[2] // 2
55 +
56 + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
57 + renormalize)
58 +
59 + # This might not be an optimal config for a single MMM
60 + get_config_func = functools.partial(try_get_optimal_moe_config,
61 + w.shape,
62 + w.shape,
63 + topk_ids.shape[1],
64 + None,
65 + override_config=override_config,
66 + is_marlin=True)
67 + config = get_config_func(M)
68 +
69 + block_size_m = config['BLOCK_SIZE_M']
70 +
71 + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
72 +
73 + max_workspace_size = (N // 64) * 16
74 + workspace = torch.zeros(max_workspace_size,
75 + dtype=torch.int,
76 + device="cuda",
77 + requires_grad=False)
78 +
79 + intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
80 + hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
81 + g_idx, perm, workspace, M, N, K, True, E, topk, block_size_m, True,
82 + False)
83 +
84 + return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
85 +
86 +
87 +def fused_marlin_moe(
88 + hidden_states: torch.Tensor,
89 + w1: torch.Tensor,
90 + w2: torch.Tensor,
91 + gating_output: torch.Tensor,
92 + g_idx1: torch.Tensor,
93 + g_idx2: torch.Tensor,
94 + perm1: torch.Tensor,
95 + perm2: torch.Tensor,
96 + topk_weights: torch.Tensor,
97 + topk_ids: torch.Tensor,
98 + override_config: Optional[Dict[str, Any]] = None,
99 + w1_scale: Optional[torch.Tensor] = None,
100 + w2_scale: Optional[torch.Tensor] = None,
101 +) -> torch.Tensor:
102 + """
103 + This function computes a Mixture of Experts (MoE) layer using two sets of
104 + weights, w1 and w2, and top-k gating mechanism.
105 +
106 + Parameters:
107 + - hidden_states (torch.Tensor): The input tensor to the MoE layer.
108 + - w1 (torch.Tensor): The first set of expert weights.
109 + - w2 (torch.Tensor): The second set of expert weights.
110 + - gating_output (torch.Tensor): The output of the gating operation
111 + (before softmax).
112 + - g_idx1 (torch.Tensor): The first set of act_order indices.
113 + - g_idx2 (torch.Tensor): The second set of act_order indices.
114 + - perm1 (torch.Tensor): The first act_order input permutation.
115 + - perm2 (torch.Tensor): The second act_order input permutation.
116 + - topk_weights (torch.Tensor): Top-k weights.
117 + - topk_ids (torch.Tensor): Indices of topk-k elements.
118 + - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
119 + - override_config (Optional[Dict[str, Any]]): Optional override
120 + for the kernel configuration.
121 + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
122 + w1.
123 + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
124 + w2.
125 +
126 + Returns:
127 + - torch.Tensor: The output tensor after applying the MoE layer.
128 + """
129 + # Check constraints.
130 + assert hidden_states.shape[0] == gating_output.shape[
131 + 0], "Number of tokens mismatch"
132 + assert hidden_states.shape[
133 + 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
134 + assert hidden_states.shape[
135 + 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
136 + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
137 + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
138 + assert w1.is_contiguous(), "Expert weights1 must be contiguous"
139 + assert w2.is_contiguous(), "Expert weights2 must be contiguous"
140 + assert hidden_states.dtype == torch.float16
141 +
142 + M, K = hidden_states.shape
143 + E = w1.shape[0]
144 + N = w2.shape[1] * 16
145 + topk = topk_ids.shape[1]
146 +
147 + get_config_func = functools.partial(
148 + try_get_optimal_moe_config,
149 + w1.shape,
150 + w2.shape,
151 + topk_ids.shape[1],
152 + None,
153 + override_config=override_config,
154 + is_marlin=True,
155 + )
156 + config = get_config_func(M)
157 +
158 + block_size_m = config["BLOCK_SIZE_M"]
159 +
160 + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
161 +
162 + max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
163 + workspace = torch.zeros(max_workspace_size,
164 + dtype=torch.int,
165 + device="cuda",
166 + requires_grad=False)
167 +
168 + intermediate_cache2 = torch.empty(
169 + (M * topk_ids.shape[1], N),
170 + device=hidden_states.device,
171 + dtype=hidden_states.dtype,
172 + )
173 +
174 + intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
175 + hidden_states,
176 + w1,
177 + sorted_token_ids,
178 + topk_weights,
179 + topk_ids,
180 + w1_scale,
181 + g_idx1,
182 + perm1,
183 + workspace,
184 + M,
185 + 2 * N,
186 + K,
187 + True,
188 + E,
189 + topk,
190 + block_size_m,
191 + True,
192 + False,
193 + )
194 +
195 + ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
196 +
197 + intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
198 + intermediate_cache2,
199 + w2,
200 + sorted_token_ids,
201 + topk_weights,
202 + topk_ids,
203 + w2_scale,
204 + g_idx2,
205 + perm2,
206 + workspace,
207 + M,
208 + K,
209 + N,
210 + True,
211 + E,
212 + topk,
213 + block_size_m,
214 + False,
215 + True,
216 + )
217 +
218 + return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
219 + dim=1)
vllm/model_executor/layers/fused_moe/fused_moe.py
+20 -118
@@ -323,15 +323,22 @@ def get_moe_configs(E: int, N: int,
323 return None
324
325
326 -def get_default_config(M: int, E: int, N: int, K: int, topk: int,
327 - dtype: Optional[str],
328 - is_marlin: bool) -> Dict[str, int]:
326 +def get_default_config(
327 + M: int,
328 + E: int,
329 + N: int,
330 + K: int,
331 + topk: int,
332 + dtype: Optional[str],
333 + is_marlin: bool,
334 +) -> Dict[str, int]:
335 config = {
336 'BLOCK_SIZE_M': 64,
337 'BLOCK_SIZE_N': 64,
338 'BLOCK_SIZE_K': 32,
339 'GROUP_SIZE_M': 8
340 }
341 + # A heuristic: fused marlin works faster with this config for small M
342 if M <= E or (is_marlin and M <= 32):
343 config = {
344 'BLOCK_SIZE_M': 16,
@@ -342,14 +349,15 @@ def get_default_config(M: int, E: int, N: int, K: int, topk: int,
349 return config
350
351
345 -def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
346 - w2_shape: Tuple[int, ...],
347 - top_k: int,
348 - dtype: Optional[str],
349 - M: int,
350 - override_config: Optional[Dict[str,
351 - Any]] = None,
352 - is_marlin: bool = False):
352 +def try_get_optimal_moe_config(
353 + w1_shape: Tuple[int, ...],
354 + w2_shape: Tuple[int, ...],
355 + top_k: int,
356 + dtype: Optional[str],
357 + M: int,
358 + override_config: Optional[Dict[str, Any]] = None,
359 + is_marlin: bool = False,
360 +):
361 if override_config:
362 config = override_config
363 else:
@@ -391,6 +399,7 @@ def fused_topk(
399 topk,
400 dtype=torch.int32,
401 device=hidden_states.device)
402 +
403 ops.topk_softmax(
404 topk_weights,
405 topk_ids,
@@ -437,113 +446,6 @@ def grouped_topk(hidden_states: torch.Tensor,
446 return topk_weights, topk_ids
447
448
440 -def fused_marlin_moe(hidden_states: torch.Tensor,
441 - w1: torch.Tensor,
442 - w2: torch.Tensor,
443 - gating_output: torch.Tensor,
444 - g_idx1: torch.Tensor,
445 - g_idx2: torch.Tensor,
446 - rand_perm1: torch.Tensor,
447 - rand_perm2: torch.Tensor,
448 - topk: int,
449 - custom_routing_function: Optional[Callable] = None,
450 - renormalize: bool = True,
451 - override_config: Optional[Dict[str, Any]] = None,
452 - use_fp8: bool = False,
453 - w1_scale: Optional[torch.Tensor] = None,
454 - w2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
455 - """
456 - This function computes a Mixture of Experts (MoE) layer using two sets of
457 - weights, w1 and w2, and top-k gating mechanism.
458 - Parameters:
459 - - hidden_states (torch.Tensor): The input tensor to the MoE layer.
460 - - w1 (torch.Tensor): The first set of expert weights.
461 - - w2 (torch.Tensor): The second set of expert weights.
462 - - gating_output (torch.Tensor): The output of the gating operation
463 - (before softmax).
464 - - topk (int): The number of top-k experts to select.
465 - - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
466 - - inplace (bool): If True, perform the operation in-place.
467 - Defaults to False.
468 - - override_config (Optional[Dict[str, Any]]): Optional override
469 - for the kernel configuration.
470 - - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
471 - products for w1 and w2. Defaults to False.
472 - - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
473 - w1.
474 - - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
475 - w2.
476 - Returns:
477 - - torch.Tensor: The output tensor after applying the MoE layer.
478 - """
479 - # Check constraints.
480 - assert hidden_states.shape[0] == gating_output.shape[0], (
481 - "Number of tokens mismatch")
482 - assert hidden_states.shape[
483 - 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
484 - assert hidden_states.shape[
485 - 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
486 - assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
487 - assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
488 - assert w1.is_contiguous(), "Expert weights1 must be contiguous"
489 - assert w2.is_contiguous(), "Expert weights2 must be contiguous"
490 - assert hidden_states.dtype in [
491 - torch.float32, torch.float16, torch.bfloat16
492 - ]
493 -
494 - #TODO fp8 is not implemented yet
495 - assert not use_fp8
496 -
497 - M, K = hidden_states.shape
498 - E = w1.shape[0]
499 - N = w2.shape[1] * 16
500 -
501 - if custom_routing_function is None:
502 - topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
503 - renormalize)
504 - else:
505 - topk_weights, topk_ids = custom_routing_function(
506 - hidden_states, gating_output, topk, renormalize)
507 -
508 - get_config_func = functools.partial(try_get_optimal_moe_config,
509 - w1.shape,
510 - w2.shape,
511 - topk_ids.shape[1],
512 - "float8" if use_fp8 else None,
513 - override_config=override_config,
514 - is_marlin=True)
515 - config = get_config_func(M)
516 -
517 - block_size_m = config['BLOCK_SIZE_M']
518 -
519 - sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
520 -
521 - max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
522 - workspace = torch.zeros(max_workspace_size,
523 - dtype=torch.int,
524 - device="cuda",
525 - requires_grad=False)
526 -
527 - intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N),
528 - device=hidden_states.device,
529 - dtype=hidden_states.dtype)
530 -
531 - intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
532 - hidden_states, w1, sorted_token_ids, topk_weights, topk_ids, w1_scale,
533 - g_idx1, rand_perm1, workspace, M, 2 * N, K, True, E, topk,
534 - block_size_m, True, False)
535 -
536 - ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
537 -
538 - intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
539 - intermediate_cache2, w2, sorted_token_ids, topk_weights, topk_ids,
540 - w2_scale, g_idx2, rand_perm2, workspace, M, K, N, True, E, topk,
541 - block_size_m, False, True)
542 -
543 - return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
544 - dim=1)
545 -
546 -
449 def get_config_dtype_str(dtype: torch.dtype,
450 use_int8_w8a16: Optional[bool] = False,
451 use_fp8_w8a8: Optional[bool] = False):
vllm/model_executor/layers/fused_moe/layer.py
+51 -24
@@ -306,10 +306,28 @@ class FusedMoE(torch.nn.Module):
306 # Input scales can be loaded directly and should be equal.
307 param_data[expert_id] = loaded_weight
308
309 + def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor,
310 + shard_dim: int, loaded_weight: torch.tensor, tp_rank: int):
311 +
312 + if shard_id == "w2":
313 + self._load_w2(shard_id=shard_id,
314 + shard_dim=shard_dim,
315 + loaded_weight=loaded_weight,
316 + expert_data=expert_data,
317 + tp_rank=tp_rank)
318 + else:
319 + assert shard_id in ("w1", "w3")
320 + expert_data.copy_(loaded_weight)
321 +
322 def weight_loader(self, param: torch.nn.Parameter,
323 loaded_weight: torch.Tensor, weight_name: str,
324 shard_id: str, expert_id: int) -> None:
325
326 + # compressed-tensors represents weights on disk which are flipped
327 + loaded_weight = loaded_weight.t().contiguous() if (
328 + self.quant_method.__class__.__name__
329 + == "CompressedTensorsMoEMethod") else loaded_weight
330 +
331 if shard_id not in ("w1", "w2", "w3"):
332 raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
333 f"got {shard_id}.")
@@ -325,19 +343,41 @@ class FusedMoE(torch.nn.Module):
343 expert_data = param.data[expert_id]
344 tp_rank = get_tensor_model_parallel_rank()
345
328 - # is_transposed: whether or not the parameter is transposed on disk
329 - # If transposed, the loaded weight will be transposed and the dim
330 - # to shard the loaded weight will be flipped.
346 + # is_transposed: if the dim to shard the weight
347 + # should be flipped. Required by GPTQ, compressed-tensors
348 + # should be whatever dimension intermediate_size is
349 is_transposed = getattr(param, "is_transposed", False)
350 shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
351 if is_transposed:
334 - loaded_weight = loaded_weight.t().contiguous()
352 shard_dim = ~shard_dim
353
337 - # Case weight_scales
338 - if "weight_scale" in weight_name:
339 - # load the weight scaling based on the quantization scheme
340 - # supported weight scales can be found in
354 + # Case input scale: input_scale loading is only supported for fp8
355 + if "input_scale" in weight_name:
356 + if param.data[expert_id] != 1 and (param.data[expert_id] -
357 + loaded_weight).abs() > 1e-5:
358 + raise ValueError(
359 + "input_scales of w1 and w3 of a layer "
360 + f"must be equal. But got {param.data[expert_id]} "
361 + f"vs. {loaded_weight}")
362 +
363 + self._load_single_value(param=param,
364 + loaded_weight=loaded_weight,
365 + expert_id=expert_id)
366 + return
367 +
368 + # Case g_idx
369 + if "g_idx" in weight_name:
370 + self._load_g_idx(shard_dim=0,
371 + shard_id=shard_id,
372 + loaded_weight=loaded_weight,
373 + expert_data=expert_data,
374 + tp_rank=tp_rank)
375 + return
376 +
377 + # Case weight scales and zero_points
378 + if ("scale" in weight_name or "zero" in weight_name):
379 + # load the weight scales and zp based on the quantization scheme
380 + # supported weight scales/zp can be found in
381 # FusedMoeWeightScaleSupported
382 # TODO @dsikka: once hardened, refactor to use vLLM Parameters
383 # specific to each case
@@ -366,22 +406,9 @@ class FusedMoE(torch.nn.Module):
406 f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
407 return
408
409 + # Case weight_shape
410 if "weight_shape" in weight_name:
370 - self._load_single_value(param=param,
371 - loaded_weight=loaded_weight,
372 - expert_id=expert_id)
373 - return
374 -
375 - # Case input scale
376 - if "input_scale" in weight_name:
377 - # Note: input_scale loading is only supported for fp8
378 - if param.data[expert_id] != 1 and (param.data[expert_id] -
379 - loaded_weight).abs() > 1e-5:
380 - raise ValueError(
381 - "input_scales of w1 and w3 of a layer "
382 - f"must be equal. But got {param.data[expert_id]} "
383 - f"vs. {loaded_weight}")
384 -
411 + # only required by compressed-tensors
412 self._load_single_value(param=param,
413 loaded_weight=loaded_weight,
414 expert_id=expert_id)
@@ -498,4 +525,4 @@ class FusedMoE(torch.nn.Module):
525 param_data[expert_id][idx] = loaded_weight
526 # If we are in the row parallel case (down_proj)
527 else:
501 - param_data[expert_id] = loaded_weight
\ No newline at end of file
528 + param_data[expert_id] = loaded_weight
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+28 -20
@@ -5,9 +5,7 @@ from typing import Callable, List, Optional
5 import torch
6
7 from vllm import _custom_ops as ops
8 -from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
9 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 - WNA16_SUPPORTED_BITS)
8 +from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
9 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10 CompressionFormat)
11 from vllm.model_executor.utils import set_weight_attrs
@@ -40,11 +38,10 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
38
39 if not (self.quant_config.quant_format
40 == CompressionFormat.pack_quantized.value
43 - and self.num_bits in WNA16_SUPPORTED_BITS):
41 + and self.num_bits == 4):
42 raise ValueError("For Fused MoE layers, only ",
43 f"{CompressionFormat.pack_quantized.value} ",
46 - "is supported for the following bits: ",
47 - f"{WNA16_SUPPORTED_BITS}")
44 + "is supported for 4 bits")
45
46 def create_weights(self, layer: torch.nn.Module, num_experts: int,
47 hidden_size: int, intermediate_size: int,
@@ -269,19 +266,30 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
266 custom_routing_function: Optional[Callable] = None,
267 ) -> torch.Tensor:
268
272 - from vllm.model_executor.layers.fused_moe.fused_moe import (
269 + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
270 fused_marlin_moe)
271
275 - return fused_marlin_moe(x,
276 - layer.w13_weight_packed,
277 - layer.w2_weight_packed,
278 - router_logits,
279 - layer.w13_g_idx,
280 - layer.w2_g_idx,
281 - layer.w13_g_idx_sort_indices,
282 - layer.w2_g_idx_sort_indices,
283 - top_k,
284 - custom_routing_function,
285 - renormalize=renormalize,
286 - w1_scale=layer.w13_weight_scale,
287 - w2_scale=layer.w2_weight_scale)
272 + topk_weights, topk_ids = FusedMoE.select_experts(
273 + hidden_states=x,
274 + router_logits=router_logits,
275 + use_grouped_topk=use_grouped_topk,
276 + top_k=top_k,
277 + renormalize=renormalize,
278 + topk_group=topk_group,
279 + num_expert_group=num_expert_group,
280 + custom_routing_function=custom_routing_function)
281 +
282 + return fused_marlin_moe(
283 + x,
284 + layer.w13_weight_packed,
285 + layer.w2_weight_packed,
286 + router_logits,
287 + layer.w13_g_idx,
288 + layer.w2_g_idx,
289 + layer.w13_g_idx_sort_indices,
290 + layer.w2_g_idx_sort_indices,
291 + topk_weights,
292 + topk_ids,
293 + w1_scale=layer.w13_weight_scale,
294 + w2_scale=layer.w2_weight_scale,
295 + )
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1 -1
@@ -22,7 +22,7 @@ from vllm.scalar_type import scalar_types
22 __all__ = ["CompressedTensorsWNA16"]
23 WNA16_SUPPORTED_TYPES_MAP = {
24 4: scalar_types.uint4b8,
25 - 8: scalar_types.uint8b128,
25 + 8: scalar_types.uint8b128
26 }
27 WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
28
vllm/model_executor/layers/quantization/gptq_marlin.py
+297 -15
@@ -1,18 +1,22 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Callable, Dict, List, Optional, Union
2
3 import torch
4 from torch.nn import Parameter
5
6 from vllm import _custom_ops as ops
7 from vllm.logger import init_logger
8 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 +from vllm.model_executor.layers.fused_moe.layer import (
9 + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 + set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
13 - marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
14 - marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
15 - verify_marlin_supported, verify_marlin_supports_shape)
16 + marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
17 + marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
18 + marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
19 + verify_marlin_supports_shape)
20 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
22 GroupQuantScaleParameter,
@@ -33,8 +37,14 @@ class GPTQMarlinConfig(QuantizationConfig):
37 (8, True): scalar_types.uint8b128,
38 }
39
36 - def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
37 - is_sym: bool, lm_head_quantized: bool) -> None:
40 + def __init__(
41 + self,
42 + weight_bits: int,
43 + group_size: int,
44 + desc_act: bool,
45 + is_sym: bool,
46 + lm_head_quantized: bool,
47 + ) -> None:
48 if desc_act and group_size == -1:
49 # In this case, act_order == True is the same as act_order == False
50 # (since we have only one group per output channel)
@@ -105,11 +115,14 @@ class GPTQMarlinConfig(QuantizationConfig):
115 " faster inference")
116 return None
117
108 - def get_quant_method(self, layer: torch.nn.Module,
109 - prefix: str) -> Optional["GPTQMarlinLinearMethod"]:
110 - if (isinstance(layer, LinearBase) or
111 - (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
118 + def get_quant_method(
119 + self, layer: torch.nn.Module, prefix: str
120 + ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod"]]:
121 + if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead)
122 + and self.lm_head_quantized):
123 return GPTQMarlinLinearMethod(self)
124 + elif isinstance(layer, FusedMoE):
125 + return GPTQMarlinMoEMethod(self)
126 return None
127
128 def get_scaled_act_names(self) -> List[str]:
@@ -179,7 +192,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
192 output_size_per_partition=output_size_per_partition,
193 input_size_per_partition=input_size_per_partition,
194 input_size=input_size,
182 - group_size=group_size)
195 + group_size=group_size,
196 + )
197
198 # Determine sharding
199 if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
@@ -299,7 +313,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
313 perm=layer.g_idx_sort_indices,
314 size_k=layer.input_size_per_partition,
315 size_n=layer.output_size_per_partition,
302 - num_bits=self.quant_config.quant_type.size_bits)
316 + num_bits=self.quant_config.quant_type.size_bits,
317 + )
318 replace_tensor(layer, "qweight", marlin_qweight)
319
320 # Permute scales from autogptq format to marlin format.
@@ -308,7 +323,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
323 size_k=(layer.input_size if self.quant_config.desc_act else
324 layer.input_size_per_partition),
325 size_n=layer.output_size_per_partition,
311 - group_size=self.quant_config.group_size)
326 + group_size=self.quant_config.group_size,
327 + )
328 replace_tensor(layer, "scales", marlin_scales)
329
330 def apply(
@@ -329,4 +345,270 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
345 output_size_per_partition=layer.output_size_per_partition,
346 input_size_per_partition=layer.input_size_per_partition,
347 is_k_full=layer.is_k_full,
332 - bias=bias)
348 + bias=bias,
349 + )
350 +
351 +
352 +class GPTQMarlinMoEMethod(FusedMoEMethodBase):
353 + """MoE Marlin method with quantization."""
354 +
355 + def __init__(self, quant_config: GPTQMarlinConfig) -> None:
356 + self.quant_config = quant_config
357 +
358 + def create_weights(
359 + self,
360 + layer: torch.nn.Module,
361 + num_experts: int,
362 + hidden_size: int,
363 + intermediate_size: int,
364 + params_dtype: torch.dtype,
365 + **extra_weight_attrs,
366 + ):
367 + # Currently assuming is_k_full is always True
368 + # (input size per partition is the same as full input size)
369 + # Supports only sym for now (no zp)
370 + if self.quant_config.group_size != -1:
371 + scales_size13 = hidden_size // self.quant_config.group_size
372 + scales_size2 = intermediate_size // self.quant_config.group_size
373 + strategy = FusedMoeWeightScaleSupported.GROUP.value
374 + else:
375 + scales_size13 = 1
376 + scales_size2 = 1
377 + strategy = FusedMoeWeightScaleSupported.CHANNEL.value
378 +
379 + extra_weight_attrs.update({
380 + "quant_method": strategy,
381 + "is_transposed": True
382 + })
383 + # Fused gate_up_proj (column parallel)
384 + w13_qweight = torch.nn.Parameter(
385 + torch.empty(
386 + num_experts,
387 + hidden_size // self.quant_config.pack_factor,
388 + 2 * intermediate_size,
389 + dtype=torch.int32,
390 + ),
391 + requires_grad=False,
392 + )
393 + layer.register_parameter("w13_qweight", w13_qweight)
394 + set_weight_attrs(w13_qweight, extra_weight_attrs)
395 + # down_proj (row parallel)
396 + w2_qweight = torch.nn.Parameter(
397 + torch.empty(
398 + num_experts,
399 + intermediate_size // self.quant_config.pack_factor,
400 + hidden_size,
401 + dtype=torch.int32,
402 + ),
403 + requires_grad=False,
404 + )
405 + layer.register_parameter("w2_qweight", w2_qweight)
406 + set_weight_attrs(w2_qweight, extra_weight_attrs)
407 + # up_proj scales
408 + w13_scales = torch.nn.Parameter(
409 + torch.empty(num_experts,
410 + scales_size13,
411 + 2 * intermediate_size,
412 + dtype=torch.half),
413 + requires_grad=False,
414 + )
415 + layer.register_parameter("w13_scales", w13_scales)
416 + set_weight_attrs(w13_scales, extra_weight_attrs)
417 + # down_proj scales
418 + w2_scales = torch.nn.Parameter(
419 + torch.empty(num_experts,
420 + scales_size2,
421 + hidden_size,
422 + dtype=torch.half),
423 + requires_grad=False,
424 + )
425 + layer.register_parameter("w2_scales", w2_scales)
426 + set_weight_attrs(w2_scales, extra_weight_attrs)
427 + # up_proj scales
428 + w13_qzeros = torch.nn.Parameter(
429 + torch.empty(num_experts,
430 + scales_size13,
431 + 2 * intermediate_size // self.quant_config.pack_factor,
432 + dtype=params_dtype),
433 + requires_grad=False,
434 + )
435 + layer.register_parameter("w13_qzeros", w13_qzeros)
436 + set_weight_attrs(w13_qzeros, extra_weight_attrs)
437 + # down_proj scales
438 + w2_qzeros = torch.nn.Parameter(
439 + torch.empty(num_experts,
440 + scales_size2,
441 + hidden_size // self.quant_config.pack_factor,
442 + dtype=params_dtype),
443 + requires_grad=False,
444 + )
445 + layer.register_parameter("w2_qzeros", w2_qzeros)
446 + set_weight_attrs(w2_qzeros, extra_weight_attrs)
447 + w13_g_idx = torch.nn.Parameter(
448 + torch.empty(
449 + num_experts,
450 + hidden_size,
451 + dtype=torch.int32,
452 + ),
453 + requires_grad=False,
454 + )
455 + layer.register_parameter("w13_g_idx", w13_g_idx)
456 + set_weight_attrs(w13_g_idx, extra_weight_attrs)
457 + w2_g_idx = torch.nn.Parameter(
458 + torch.empty(
459 + num_experts,
460 + intermediate_size,
461 + dtype=torch.int32,
462 + ),
463 + requires_grad=False,
464 + )
465 + layer.register_parameter("w2_g_idx", w2_g_idx)
466 + set_weight_attrs(w2_g_idx, extra_weight_attrs)
467 + w13_g_idx_sort_indices = torch.nn.Parameter(
468 + torch.empty(
469 + num_experts,
470 + hidden_size,
471 + dtype=torch.int32,
472 + ),
473 + requires_grad=False,
474 + )
475 + layer.register_parameter("w13_g_idx_sort_indices",
476 + w13_g_idx_sort_indices)
477 + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
478 + w2_g_idx_sort_indices = torch.nn.Parameter(
479 + torch.empty(
480 + num_experts,
481 + intermediate_size,
482 + dtype=torch.int32,
483 + ),
484 + requires_grad=False,
485 + )
486 + layer.register_parameter("w2_g_idx_sort_indices",
487 + w2_g_idx_sort_indices)
488 + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
489 +
490 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
491 +
492 + # Process act_order
493 + if self.quant_config.desc_act:
494 + # Get sorting based on g_idx
495 + num_experts = layer.w13_g_idx.shape[0]
496 + w13_g_idx_sort_indices = torch.empty_like(layer.w13_g_idx)
497 + w2_g_idx_sort_indices = torch.empty_like(layer.w2_g_idx)
498 + w13_sorted_g_idx = torch.empty_like(layer.w13_g_idx)
499 + w2_sorted_g_idx = torch.empty_like(layer.w2_g_idx)
500 + for e in range(num_experts):
501 + w13_g_idx_sort_indices[e] = torch.argsort(
502 + layer.w13_g_idx[e]).to(torch.int32)
503 + w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_g_idx[e]).to(
504 + torch.int32)
505 + w13_sorted_g_idx[e] = layer.w13_g_idx[e][
506 + w13_g_idx_sort_indices[e]]
507 + w2_sorted_g_idx[e] = layer.w2_g_idx[e][
508 + w2_g_idx_sort_indices[e]]
509 + replace_tensor(layer, "w13_g_idx", w13_sorted_g_idx)
510 + replace_tensor(layer, "w2_g_idx", w2_sorted_g_idx)
511 + replace_tensor(layer, "w13_g_idx_sort_indices",
512 + w13_g_idx_sort_indices)
513 + replace_tensor(layer, "w2_g_idx_sort_indices",
514 + w2_g_idx_sort_indices)
515 + else:
516 + # Reset g_idx related tensors
517 + num_experts = layer.w13_g_idx.shape[0]
518 + device = layer.w13_g_idx.device
519 + layer.w13_g_idx = torch.nn.Parameter(
520 + torch.empty((num_experts, 0), dtype=torch.int32,
521 + device=device),
522 + requires_grad=False,
523 + )
524 + layer.w2_g_idx = torch.nn.Parameter(
525 + torch.empty((num_experts, 0), dtype=torch.int32,
526 + device=device),
527 + requires_grad=False,
528 + )
529 + layer.w13_g_idx_sort_indices = torch.nn.Parameter(
530 + torch.empty((num_experts, 0), dtype=torch.int32,
531 + device=device),
532 + requires_grad=False,
533 + )
534 + layer.w2_g_idx_sort_indices = torch.nn.Parameter(
535 + torch.empty((num_experts, 0), dtype=torch.int32,
536 + device=device),
537 + requires_grad=False,
538 + )
539 + # Repack weights
540 + marlin_w13_qweight = ops.gptq_marlin_moe_repack(
541 + layer.w13_qweight,
542 + layer.w13_g_idx_sort_indices,
543 + layer.w13_qweight.shape[1] * self.quant_config.pack_factor,
544 + layer.w13_qweight.shape[2],
545 + self.quant_config.quant_type.size_bits,
546 + )
547 + replace_tensor(layer, "w13_qweight", marlin_w13_qweight)
548 + marlin_w2_qweight = ops.gptq_marlin_moe_repack(
549 + layer.w2_qweight,
550 + layer.w2_g_idx_sort_indices,
551 + layer.w2_qweight.shape[1] * self.quant_config.pack_factor,
552 + layer.w2_qweight.shape[2],
553 + self.quant_config.quant_type.size_bits,
554 + )
555 + replace_tensor(layer, "w2_qweight", marlin_w2_qweight)
556 + # Repack scales
557 + marlin_w13_scales = marlin_moe_permute_scales(
558 + s=layer.w13_scales,
559 + size_k=layer.intermediate_size_per_partition,
560 + size_n=layer.w13_scales.shape[2],
561 + group_size=self.quant_config.group_size,
562 + )
563 + replace_tensor(layer, "w13_scales", marlin_w13_scales)
564 + marlin_w2_scales = marlin_moe_permute_scales(
565 + s=layer.w2_scales,
566 + size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor,
567 + size_n=layer.w2_scales.shape[2],
568 + group_size=self.quant_config.group_size,
569 + )
570 + replace_tensor(layer, "w2_scales", marlin_w2_scales)
571 +
572 + def apply(
573 + self,
574 + layer: torch.nn.Module,
575 + x: torch.Tensor,
576 + router_logits: torch.Tensor,
577 + top_k: int,
578 + renormalize: bool = True,
579 + use_grouped_topk: bool = False,
580 + num_expert_group: Optional[int] = None,
581 + topk_group: Optional[int] = None,
582 + custom_routing_function: Optional[Callable] = None,
583 + ) -> torch.Tensor:
584 + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
585 + fused_marlin_moe)
586 +
587 + # The input must currently be float16
588 + orig_dtype = x.dtype
589 + x = x.half()
590 +
591 + topk_weights, topk_ids = FusedMoE.select_experts(
592 + hidden_states=x,
593 + router_logits=router_logits,
594 + use_grouped_topk=use_grouped_topk,
595 + top_k=top_k,
596 + renormalize=renormalize,
597 + topk_group=topk_group,
598 + num_expert_group=num_expert_group,
599 + custom_routing_function=None)
600 +
601 + return fused_marlin_moe(
602 + x,
603 + layer.w13_qweight,
604 + layer.w2_qweight,
605 + router_logits,
606 + layer.w13_g_idx,
607 + layer.w2_g_idx,
608 + layer.w13_g_idx_sort_indices,
609 + layer.w2_g_idx_sort_indices,
610 + topk_weights,
611 + topk_ids,
612 + w1_scale=layer.w13_scales,
613 + w2_scale=layer.w2_scales,
614 + ).to(orig_dtype)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+17
@@ -176,6 +176,23 @@ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
176 return s
177
178
179 +def marlin_moe_permute_scales(
180 + s: torch.Tensor,
181 + size_k: int,
182 + size_n: int,
183 + group_size: int,
184 +):
185 + num_experts = s.shape[0]
186 + output = torch.empty(
187 + (num_experts, s.shape[1], s.shape[2]),
188 + device=s.device,
189 + dtype=s.dtype,
190 + )
191 + for e in range(num_experts):
192 + output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size)
193 + return output
194 +
195 +
196 def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
197 num_bits: int) -> torch.Tensor:
198 # Permute zero-points in a similar way to scales, but do not use the
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+7 -4
@@ -1,6 +1,6 @@
1 """Utility functions used for tests and benchmarks"""
2
3 -from typing import List
3 +from typing import List, Optional
4
5 import numpy as np
6 import torch
@@ -92,8 +92,11 @@ def get_weight_perm(num_bits: int):
92 return perm
93
94
95 -def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
96 - act_order: bool):
95 +def marlin_quantize(w: torch.Tensor,
96 + quant_type: ScalarType,
97 + group_size: int,
98 + act_order: bool,
99 + test_perm: Optional[torch.Tensor] = None):
100 size_k, size_n = w.shape
101 num_bits = quant_type.size_bits
102
@@ -104,7 +107,7 @@ def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
107
108 # Quantize (and apply act_order if provided)
109 w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
107 - w, quant_type, group_size, act_order)
110 + w, quant_type, group_size, act_order, test_perm)
111
112 # For act_order, sort the "weights" and "g_idx" so that group ids are
113 # increasing
vllm/model_executor/layers/quantization/utils/quant_utils.py
+13 -6
@@ -1,5 +1,5 @@
1 """This file is used for /tests and /benchmarks"""
2 -from typing import List
2 +from typing import List, Optional
3
4 import numpy
5 import torch
@@ -53,7 +53,10 @@ def get_pack_factor(num_bits):
53 return 32 // num_bits
54
55
56 -def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
56 +def permute_rows(q_w: torch.Tensor,
57 + w_ref: torch.Tensor,
58 + group_size: int,
59 + test_perm: Optional[torch.Tensor] = None):
60 assert q_w.shape == w_ref.shape
61
62 orig_device = q_w.device
@@ -64,7 +67,7 @@ def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
67 g_idx[i] = i // group_size
68
69 # Simulate act_order by doing a random permutation on K
67 - rand_perm = torch.randperm(k_size)
70 + rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
71
72 g_idx = g_idx[rand_perm].contiguous()
73 q_w = q_w[rand_perm, :].contiguous()
@@ -164,8 +167,11 @@ def quantize_weights(w: torch.Tensor,
167 )
168
169
167 -def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
168 - group_size: int, act_order: bool):
170 +def gptq_quantize_weights(w: torch.Tensor,
171 + quant_type: ScalarType,
172 + group_size: int,
173 + act_order: bool,
174 + test_perm: Optional[torch.Tensor] = None):
175 size_k, _ = w.shape
176
177 assert w.is_floating_point(), "w must be float"
@@ -186,7 +192,8 @@ def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
192 ), "For act_order, groupsize = {} must be less than size_k = {}".format(
193 group_size, size_k)
194
189 - w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size)
195 + w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size,
196 + test_perm)
197
198 return w_ref, w_q, w_s, g_idx, rand_perm
199
vllm/model_executor/model_loader/utils.py
+8
@@ -24,10 +24,18 @@ def get_model_architecture(
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 mixtral_supported = ["fp8", "compressed-tensors"]
27 + # for gptq_marlin, only run fused MoE for int4
28 + if model_config.quantization == "gptq_marlin":
29 + hf_quant_config = getattr(model_config.hf_config,
30 + "quantization_config", None)
31 + if hf_quant_config and hf_quant_config.get("bits") == 4:
32 + mixtral_supported.append("gptq_marlin")
33 +
34 if (model_config.quantization is not None
35 and model_config.quantization not in mixtral_supported
36 and "MixtralForCausalLM" in architectures):
37 architectures = ["QuantMixtralForCausalLM"]
38 +
39 return ModelRegistry.resolve_model_cls(architectures)
40
41
vllm/model_executor/models/mixtral.py
+7 -2
@@ -435,7 +435,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
435 continue
436 name = name.replace(weight_name, param_name)
437 # Skip loading extra bias for GPTQ models.
438 - if name.endswith(".bias") and name not in params_dict:
438 + if ((name.endswith(".bias") or name.endswith("_bias"))
439 + and name not in params_dict):
440 continue
441 # Skip layers on other devices.
442 if is_pp_missing_parameter(name, self):
@@ -454,6 +455,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
455 # Skip layers on other devices.
456 if is_pp_missing_parameter(name, self):
457 continue
458 + if ((name.endswith(".bias") or name.endswith("_bias"))
459 + and name not in params_dict):
460 + continue
461 param = params_dict[name]
462 weight_loader = param.weight_loader
463 weight_loader(param,
@@ -464,7 +468,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
468 break
469 else:
470 # Skip loading extra bias for GPTQ models.
467 - if name.endswith(".bias") and name not in params_dict:
471 + if ((name.endswith(".bias") or name.endswith("_bias"))
472 + and name not in params_dict):
473 continue
474 # Skip layers on other devices.
475 if is_pp_missing_parameter(name, self):