A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Fused MoE Marlin support for GPTQ (#8217)
Dipika Sikka committed
Sep 9, 2024 at 23:02 UTC
6cd5e5b07e4415d064d93b8a66331a097bd9287e
19 files changed
+912
-204
.buildkite/test-pipeline.yaml
+12
-1
@@ -386,7 +386,18 @@ steps:
386
- vllm/
387
- tests/weight_loading
388
commands:
389
- - bash weight_loading/run_model_weight_loading_test.sh
389
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
390
+
391
+- label: Weight Loading Multiple GPU Test - Large Models # optional
392
+ working_dir: "/vllm-workspace/tests"
393
+ num_gpus: 2
394
+ gpu: a100
395
+ optional: true
396
+ source_file_dependencies:
397
+ - vllm/
398
+ - tests/weight_loading
399
+ commands:
400
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt
401
402
403
##### multi gpus test #####
csrc/moe/marlin_moe_ops.cu
+1
-1
@@ -1737,4 +1737,4 @@ torch::Tensor marlin_gemm_moe(
1737
moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1738
thread_n, sms, max_par, replicate_input, apply_weights);
1739
return c;
1740
-}
\ No newline at end of file
1740
+}
csrc/moe/marlin_moe_ops.h
+1
-1
@@ -9,4 +9,4 @@ torch::Tensor marlin_gemm_moe(
9
const torch::Tensor& g_idx, const torch::Tensor& perm,
10
torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11
bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12
- bool replicate_input, bool apply_weights);
\ No newline at end of file
12
+ bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
-1
@@ -16,7 +16,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
16
"g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
17
"size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
18
"bool replicate_input, bool apply_weights) -> Tensor");
19
-
19
m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
20
#endif
21
}
tests/kernels/test_moe.py
+217
-4
@@ -2,6 +2,8 @@
2
3
Run `pytest tests/kernels/test_moe.py`.
4
"""
5
+from typing import List
6
+
7
import pytest
8
import torch
9
from transformers import MixtralConfig
@@ -9,7 +11,13 @@ from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
11
12
from vllm.model_executor.layers.activation import SiluAndMul
13
from vllm.model_executor.layers.fused_moe import fused_moe
14
+from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15
+ fused_marlin_moe, single_marlin_moe)
16
+from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
17
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
18
+ marlin_quantize)
19
from vllm.model_executor.models.mixtral import MixtralMoE
20
+from vllm.scalar_type import scalar_types
21
22
23
def torch_moe(a, w1, w2, score, topk):
@@ -29,6 +37,20 @@ def torch_moe(a, w1, w2, score, topk):
37
topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
38
39
40
+def torch_moe_single(a, w, score, topk):
41
+ B, D = a.shape
42
+ a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
43
+ out = torch.zeros(B * topk, w.shape[1], dtype=a.dtype, device=a.device)
44
+ score = torch.softmax(score, dim=-1, dtype=torch.float32)
45
+ _, topk_ids = torch.topk(score, topk)
46
+ topk_ids = topk_ids.view(-1)
47
+ for i in range(w.shape[0]):
48
+ mask = topk_ids == i
49
+ if mask.sum():
50
+ out[mask] = a[mask] @ w[i].transpose(0, 1)
51
+ return (out.view(B, -1, w.shape[1])).sum(dim=1)
52
+
53
+
54
@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
55
@pytest.mark.parametrize("n", [2048, 256, 1024])
56
@pytest.mark.parametrize("k", [128, 511, 1024])
@@ -43,11 +65,11 @@ def test_fused_moe(
65
topk: int,
66
dtype: torch.dtype,
67
):
46
- a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
47
- w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
48
- w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
68
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
69
+ w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
70
+ w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
71
50
- score = torch.randn((m, e), device='cuda', dtype=dtype)
72
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
73
triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
74
torch_output = torch_moe(a, w1, w2, score, topk)
75
torch.testing.assert_close(triton_output, torch_output, atol=1e-2, rtol=0)
@@ -99,3 +121,194 @@ def test_mixtral_moe(dtype: torch.dtype):
121
vllm_states,
122
rtol=mixtral_moe_tol[dtype],
123
atol=mixtral_moe_tol[dtype])
124
+
125
+
126
+def stack_and_dev(tensors: List[torch.Tensor]):
127
+ dev = tensors[0].device
128
+ return torch.stack(tensors, dim=0).to(dev)
129
+
130
+
131
+def compute_max_diff(output, output_ref):
132
+ return torch.mean(torch.abs(output - output_ref)) / torch.mean(
133
+ torch.abs(output_ref))
134
+
135
+
136
+@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
137
+@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
138
+@pytest.mark.parametrize("k", [128, 1024, 512])
139
+@pytest.mark.parametrize("e", [4, 8, 64])
140
+@pytest.mark.parametrize("topk", [2, 6])
141
+@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
142
+@pytest.mark.parametrize("act_order", [True, False])
143
+def test_fused_marlin_moe(
144
+ m: int,
145
+ n: int,
146
+ k: int,
147
+ e: int,
148
+ topk: int,
149
+ group_size: int,
150
+ act_order: bool,
151
+):
152
+ torch.manual_seed(7)
153
+
154
+ if topk > e:
155
+ return
156
+
157
+ # Filter act_order
158
+ if act_order:
159
+ if group_size == -1:
160
+ return
161
+ if group_size in (k, n):
162
+ return
163
+
164
+ quant_type = scalar_types.uint4b8
165
+ dtype = torch.float16
166
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
167
+ w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
168
+ w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
169
+ for i in range(w2.shape[0]):
170
+ w2[0] = torch.eye(k, n, device="cuda", dtype=dtype)
171
+
172
+ w_ref1_l = []
173
+ qweight1_l = []
174
+ scales1_l = []
175
+ g_idx1_l = []
176
+ sort_indices1_l = []
177
+
178
+ for i in range(w1.shape[0]):
179
+ test_perm = torch.randperm(k)
180
+ w_ref1, qweight1, scales1, g_idx1, sort_indices1, _ = marlin_quantize(
181
+ w1[i].transpose(1, 0), quant_type, group_size, act_order,
182
+ test_perm)
183
+ w_ref1_l.append(w_ref1)
184
+ qweight1_l.append(qweight1)
185
+ scales1_l.append(scales1)
186
+ g_idx1_l.append(g_idx1)
187
+ sort_indices1_l.append(sort_indices1)
188
+
189
+ w_ref1 = stack_and_dev(w_ref1_l)
190
+ qweight1 = stack_and_dev(qweight1_l).contiguous()
191
+ scales1 = stack_and_dev(scales1_l)
192
+ g_idx1 = stack_and_dev(g_idx1_l)
193
+ sort_indices1 = stack_and_dev(sort_indices1_l)
194
+
195
+ w_ref2_l = []
196
+ qweight2_l = []
197
+ scales2_l = []
198
+ g_idx2_l = []
199
+ sort_indices2_l = []
200
+
201
+ for i in range(w2.shape[0]):
202
+ test_perm = torch.randperm(n)
203
+ w_ref2, qweight2, scales2, g_idx2, sort_indices2, _ = marlin_quantize(
204
+ w2[i].transpose(1, 0), quant_type, group_size, act_order,
205
+ test_perm)
206
+ w_ref2_l.append(w_ref2)
207
+ qweight2_l.append(qweight2)
208
+ scales2_l.append(scales2)
209
+ g_idx2_l.append(g_idx2)
210
+ sort_indices2_l.append(sort_indices2)
211
+
212
+ w_ref2 = stack_and_dev(w_ref2_l)
213
+ qweight2 = stack_and_dev(qweight2_l).contiguous()
214
+ scales2 = stack_and_dev(scales2_l)
215
+ g_idx2 = stack_and_dev(g_idx2_l)
216
+ sort_indices2 = stack_and_dev(sort_indices2_l)
217
+
218
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
219
+
220
+ topk_weights, topk_ids = fused_topk(a, score, topk, False)
221
+
222
+ triton_output = fused_moe(
223
+ a,
224
+ w_ref1.transpose(1, 2).contiguous(),
225
+ w_ref2.transpose(1, 2).contiguous(),
226
+ score,
227
+ topk,
228
+ renormalize=False,
229
+ )
230
+ marlin_output = fused_marlin_moe(
231
+ a,
232
+ qweight1,
233
+ qweight2,
234
+ score,
235
+ g_idx1,
236
+ g_idx2,
237
+ sort_indices1,
238
+ sort_indices2,
239
+ topk_weights,
240
+ topk_ids,
241
+ w1_scale=scales1,
242
+ w2_scale=scales2,
243
+ )
244
+
245
+ assert compute_max_diff(marlin_output, triton_output) < 4e-2
246
+
247
+
248
+@pytest.mark.skip("This test is here for the sake of debugging, "
249
+ "don't run it in automated tests.")
250
+@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
251
+@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
252
+@pytest.mark.parametrize("k", [128, 1024, 512])
253
+@pytest.mark.parametrize("e", [4, 8, 64])
254
+@pytest.mark.parametrize("topk", [2, 6])
255
+@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
256
+@pytest.mark.parametrize("act_order", [True, False])
257
+def test_marlin_moe_mmm(
258
+ m: int,
259
+ n: int,
260
+ k: int,
261
+ e: int,
262
+ topk: int,
263
+ group_size: int,
264
+ act_order: bool,
265
+):
266
+ if topk > e:
267
+ return
268
+
269
+ # Filter act_order
270
+ if act_order:
271
+ if group_size == -1:
272
+ return
273
+ if group_size == k:
274
+ return
275
+
276
+ quant_type = scalar_types.uint4b8
277
+ dtype = torch.float16
278
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
279
+ w = torch.randn((e, n, k), device="cuda", dtype=dtype) / 10
280
+
281
+ w_ref_l = []
282
+ qweights_l = []
283
+ scales_l = []
284
+ g_idx_l = []
285
+ sort_indices_l = []
286
+
287
+ for i in range(w.shape[0]):
288
+ test_perm = torch.randperm(k)
289
+ w_ref, qweight, scales, g_idx, sort_indices, _ = marlin_quantize(
290
+ w[i].transpose(1, 0), quant_type, group_size, act_order, test_perm)
291
+ w_ref_l.append(w_ref)
292
+ qweights_l.append(qweight)
293
+ scales_l.append(scales)
294
+ g_idx_l.append(g_idx)
295
+ sort_indices_l.append(sort_indices)
296
+
297
+ w_ref = stack_and_dev(w_ref_l)
298
+ qweight = stack_and_dev(qweights_l).contiguous()
299
+ scales = stack_and_dev(scales_l)
300
+ g_idx = stack_and_dev(g_idx_l)
301
+ sort_indices = stack_and_dev(sort_indices_l)
302
+
303
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
304
+ marlin_output = single_marlin_moe(a,
305
+ qweight,
306
+ scales,
307
+ score,
308
+ g_idx,
309
+ sort_indices,
310
+ topk,
311
+ renormalize=False)
312
+ torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
313
+
314
+ assert compute_max_diff(marlin_output, torch_output) < 1e-2
tests/weight_loading/models-large.txt
new
+3
@@ -0,0 +1,3 @@
1
+compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2
+compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3
+gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
\ No newline at end of file
tests/weight_loading/models.txt
-2
@@ -19,8 +19,6 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
19
compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
20
compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22
-compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
23
-compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
22
compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23
awq, casperhansen/mixtral-instruct-awq, main
24
awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/model_executor/layers/fused_moe/__init__.py
+10
-4
@@ -2,16 +2,22 @@ from vllm.model_executor.layers.fused_moe.layer import (
2
FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
3
from vllm.triton_utils import HAS_TRITON
4
5
-__all__ = ["FusedMoE", "FusedMoEMethodBase", "FusedMoeWeightScaleSupported"]
5
+__all__ = [
6
+ "FusedMoE",
7
+ "FusedMoEMethodBase",
8
+ "FusedMoeWeightScaleSupported",
9
+]
10
11
if HAS_TRITON:
8
-
12
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
13
+ fused_marlin_moe, single_marlin_moe)
14
from vllm.model_executor.layers.fused_moe.fused_moe import (
10
- fused_experts, fused_marlin_moe, fused_moe, fused_topk,
11
- get_config_file_name, grouped_topk)
15
+ fused_experts, fused_moe, fused_topk, get_config_file_name,
16
+ grouped_topk)
17
18
__all__ += [
19
"fused_marlin_moe",
20
+ "single_marlin_moe",
21
"fused_moe",
22
"fused_topk",
23
"fused_experts",
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
new
+219
@@ -0,0 +1,219 @@
1
+"""Fused MoE utilities for GPTQ."""
2
+import functools
3
+from typing import Any, Dict, Optional
4
+
5
+import torch
6
+
7
+from vllm import _custom_ops as ops
8
+from vllm.model_executor.layers.fused_moe.fused_moe import (
9
+ fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10
+
11
+
12
+def single_marlin_moe(
13
+ hidden_states: torch.Tensor,
14
+ w: torch.Tensor,
15
+ scales: torch.Tensor,
16
+ gating_output: torch.Tensor,
17
+ g_idx: torch.Tensor,
18
+ perm: torch.Tensor,
19
+ topk: int,
20
+ renormalize: bool,
21
+ override_config: Optional[Dict[str, Any]] = None) -> torch.Tensor:
22
+ """
23
+ This function computes the multiplication of hidden_states with expert
24
+ weights used in Marlin MoE, using weights w and top-k gating mechanism.
25
+ Its purpose is testing and debugging the fused MoE kernel.
26
+
27
+ Parameters:
28
+ - hidden_states (torch.Tensor): The input tensor to the Marlin Mul.
29
+ - w (torch.Tensor): The set of expert weights.
30
+ - scales (torch.Tensor): The quantization scales.
31
+ - gating_output (torch.Tensor): The output of the gating operation
32
+ (before softmax).
33
+ - g_idx (torch.Tensor): The act_order indices.
34
+ - perm (torch.Tensor): The act_order input permutation.
35
+ - topk (int): The number of top-k experts to select.
36
+ - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
37
+ - override_config (Optional[Dict[str, Any]]): Optional override
38
+ for the kernel configuration.
39
+
40
+ Returns:
41
+ - torch.Tensor: The output tensor after applying the MoE layer.
42
+ """
43
+ # Check constraints.
44
+ assert hidden_states.shape[0] == gating_output.shape[0], (
45
+ "Number of tokens mismatch")
46
+ assert hidden_states.shape[1] == w.shape[1] * 16, "Hidden size mismatch"
47
+ assert gating_output.shape[1] == w.shape[0], "Number of experts mismatch"
48
+ assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
49
+ assert w.is_contiguous(), "Expert weights must be contiguous"
50
+ assert hidden_states.dtype == torch.float16
51
+
52
+ M, K = hidden_states.shape
53
+ E = w.shape[0]
54
+ N = w.shape[2] // 2
55
+
56
+ topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
57
+ renormalize)
58
+
59
+ # This might not be an optimal config for a single MMM
60
+ get_config_func = functools.partial(try_get_optimal_moe_config,
61
+ w.shape,
62
+ w.shape,
63
+ topk_ids.shape[1],
64
+ None,
65
+ override_config=override_config,
66
+ is_marlin=True)
67
+ config = get_config_func(M)
68
+
69
+ block_size_m = config['BLOCK_SIZE_M']
70
+
71
+ sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
72
+
73
+ max_workspace_size = (N // 64) * 16
74
+ workspace = torch.zeros(max_workspace_size,
75
+ dtype=torch.int,
76
+ device="cuda",
77
+ requires_grad=False)
78
+
79
+ intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
80
+ hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
81
+ g_idx, perm, workspace, M, N, K, True, E, topk, block_size_m, True,
82
+ False)
83
+
84
+ return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
85
+
86
+
87
+def fused_marlin_moe(
88
+ hidden_states: torch.Tensor,
89
+ w1: torch.Tensor,
90
+ w2: torch.Tensor,
91
+ gating_output: torch.Tensor,
92
+ g_idx1: torch.Tensor,
93
+ g_idx2: torch.Tensor,
94
+ perm1: torch.Tensor,
95
+ perm2: torch.Tensor,
96
+ topk_weights: torch.Tensor,
97
+ topk_ids: torch.Tensor,
98
+ override_config: Optional[Dict[str, Any]] = None,
99
+ w1_scale: Optional[torch.Tensor] = None,
100
+ w2_scale: Optional[torch.Tensor] = None,
101
+) -> torch.Tensor:
102
+ """
103
+ This function computes a Mixture of Experts (MoE) layer using two sets of
104
+ weights, w1 and w2, and top-k gating mechanism.
105
+
106
+ Parameters:
107
+ - hidden_states (torch.Tensor): The input tensor to the MoE layer.
108
+ - w1 (torch.Tensor): The first set of expert weights.
109
+ - w2 (torch.Tensor): The second set of expert weights.
110
+ - gating_output (torch.Tensor): The output of the gating operation
111
+ (before softmax).
112
+ - g_idx1 (torch.Tensor): The first set of act_order indices.
113
+ - g_idx2 (torch.Tensor): The second set of act_order indices.
114
+ - perm1 (torch.Tensor): The first act_order input permutation.
115
+ - perm2 (torch.Tensor): The second act_order input permutation.
116
+ - topk_weights (torch.Tensor): Top-k weights.
117
+ - topk_ids (torch.Tensor): Indices of topk-k elements.
118
+ - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
119
+ - override_config (Optional[Dict[str, Any]]): Optional override
120
+ for the kernel configuration.
121
+ - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
122
+ w1.
123
+ - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
124
+ w2.
125
+
126
+ Returns:
127
+ - torch.Tensor: The output tensor after applying the MoE layer.
128
+ """
129
+ # Check constraints.
130
+ assert hidden_states.shape[0] == gating_output.shape[
131
+ 0], "Number of tokens mismatch"
132
+ assert hidden_states.shape[
133
+ 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
134
+ assert hidden_states.shape[
135
+ 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
136
+ assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
137
+ assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
138
+ assert w1.is_contiguous(), "Expert weights1 must be contiguous"
139
+ assert w2.is_contiguous(), "Expert weights2 must be contiguous"
140
+ assert hidden_states.dtype == torch.float16
141
+
142
+ M, K = hidden_states.shape
143
+ E = w1.shape[0]
144
+ N = w2.shape[1] * 16
145
+ topk = topk_ids.shape[1]
146
+
147
+ get_config_func = functools.partial(
148
+ try_get_optimal_moe_config,
149
+ w1.shape,
150
+ w2.shape,
151
+ topk_ids.shape[1],
152
+ None,
153
+ override_config=override_config,
154
+ is_marlin=True,
155
+ )
156
+ config = get_config_func(M)
157
+
158
+ block_size_m = config["BLOCK_SIZE_M"]
159
+
160
+ sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
161
+
162
+ max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
163
+ workspace = torch.zeros(max_workspace_size,
164
+ dtype=torch.int,
165
+ device="cuda",
166
+ requires_grad=False)
167
+
168
+ intermediate_cache2 = torch.empty(
169
+ (M * topk_ids.shape[1], N),
170
+ device=hidden_states.device,
171
+ dtype=hidden_states.dtype,
172
+ )
173
+
174
+ intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
175
+ hidden_states,
176
+ w1,
177
+ sorted_token_ids,
178
+ topk_weights,
179
+ topk_ids,
180
+ w1_scale,
181
+ g_idx1,
182
+ perm1,
183
+ workspace,
184
+ M,
185
+ 2 * N,
186
+ K,
187
+ True,
188
+ E,
189
+ topk,
190
+ block_size_m,
191
+ True,
192
+ False,
193
+ )
194
+
195
+ ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
196
+
197
+ intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
198
+ intermediate_cache2,
199
+ w2,
200
+ sorted_token_ids,
201
+ topk_weights,
202
+ topk_ids,
203
+ w2_scale,
204
+ g_idx2,
205
+ perm2,
206
+ workspace,
207
+ M,
208
+ K,
209
+ N,
210
+ True,
211
+ E,
212
+ topk,
213
+ block_size_m,
214
+ False,
215
+ True,
216
+ )
217
+
218
+ return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
219
+ dim=1)
vllm/model_executor/layers/fused_moe/fused_moe.py
+20
-118
@@ -323,15 +323,22 @@ def get_moe_configs(E: int, N: int,
323
return None
324
325
326
-def get_default_config(M: int, E: int, N: int, K: int, topk: int,
327
- dtype: Optional[str],
328
- is_marlin: bool) -> Dict[str, int]:
326
+def get_default_config(
327
+ M: int,
328
+ E: int,
329
+ N: int,
330
+ K: int,
331
+ topk: int,
332
+ dtype: Optional[str],
333
+ is_marlin: bool,
334
+) -> Dict[str, int]:
335
config = {
336
'BLOCK_SIZE_M': 64,
337
'BLOCK_SIZE_N': 64,
338
'BLOCK_SIZE_K': 32,
339
'GROUP_SIZE_M': 8
340
}
341
+ # A heuristic: fused marlin works faster with this config for small M
342
if M <= E or (is_marlin and M <= 32):
343
config = {
344
'BLOCK_SIZE_M': 16,
@@ -342,14 +349,15 @@ def get_default_config(M: int, E: int, N: int, K: int, topk: int,
349
return config
350
351
345
-def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
346
- w2_shape: Tuple[int, ...],
347
- top_k: int,
348
- dtype: Optional[str],
349
- M: int,
350
- override_config: Optional[Dict[str,
351
- Any]] = None,
352
- is_marlin: bool = False):
352
+def try_get_optimal_moe_config(
353
+ w1_shape: Tuple[int, ...],
354
+ w2_shape: Tuple[int, ...],
355
+ top_k: int,
356
+ dtype: Optional[str],
357
+ M: int,
358
+ override_config: Optional[Dict[str, Any]] = None,
359
+ is_marlin: bool = False,
360
+):
361
if override_config:
362
config = override_config
363
else:
@@ -391,6 +399,7 @@ def fused_topk(
399
topk,
400
dtype=torch.int32,
401
device=hidden_states.device)
402
+
403
ops.topk_softmax(
404
topk_weights,
405
topk_ids,
@@ -437,113 +446,6 @@ def grouped_topk(hidden_states: torch.Tensor,
446
return topk_weights, topk_ids
447
448
440
-def fused_marlin_moe(hidden_states: torch.Tensor,
441
- w1: torch.Tensor,
442
- w2: torch.Tensor,
443
- gating_output: torch.Tensor,
444
- g_idx1: torch.Tensor,
445
- g_idx2: torch.Tensor,
446
- rand_perm1: torch.Tensor,
447
- rand_perm2: torch.Tensor,
448
- topk: int,
449
- custom_routing_function: Optional[Callable] = None,
450
- renormalize: bool = True,
451
- override_config: Optional[Dict[str, Any]] = None,
452
- use_fp8: bool = False,
453
- w1_scale: Optional[torch.Tensor] = None,
454
- w2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
455
- """
456
- This function computes a Mixture of Experts (MoE) layer using two sets of
457
- weights, w1 and w2, and top-k gating mechanism.
458
- Parameters:
459
- - hidden_states (torch.Tensor): The input tensor to the MoE layer.
460
- - w1 (torch.Tensor): The first set of expert weights.
461
- - w2 (torch.Tensor): The second set of expert weights.
462
- - gating_output (torch.Tensor): The output of the gating operation
463
- (before softmax).
464
- - topk (int): The number of top-k experts to select.
465
- - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
466
- - inplace (bool): If True, perform the operation in-place.
467
- Defaults to False.
468
- - override_config (Optional[Dict[str, Any]]): Optional override
469
- for the kernel configuration.
470
- - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
471
- products for w1 and w2. Defaults to False.
472
- - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
473
- w1.
474
- - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
475
- w2.
476
- Returns:
477
- - torch.Tensor: The output tensor after applying the MoE layer.
478
- """
479
- # Check constraints.
480
- assert hidden_states.shape[0] == gating_output.shape[0], (
481
- "Number of tokens mismatch")
482
- assert hidden_states.shape[
483
- 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
484
- assert hidden_states.shape[
485
- 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
486
- assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
487
- assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
488
- assert w1.is_contiguous(), "Expert weights1 must be contiguous"
489
- assert w2.is_contiguous(), "Expert weights2 must be contiguous"
490
- assert hidden_states.dtype in [
491
- torch.float32, torch.float16, torch.bfloat16
492
- ]
493
-
494
- #TODO fp8 is not implemented yet
495
- assert not use_fp8
496
-
497
- M, K = hidden_states.shape
498
- E = w1.shape[0]
499
- N = w2.shape[1] * 16
500
-
501
- if custom_routing_function is None:
502
- topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
503
- renormalize)
504
- else:
505
- topk_weights, topk_ids = custom_routing_function(
506
- hidden_states, gating_output, topk, renormalize)
507
-
508
- get_config_func = functools.partial(try_get_optimal_moe_config,
509
- w1.shape,
510
- w2.shape,
511
- topk_ids.shape[1],
512
- "float8" if use_fp8 else None,
513
- override_config=override_config,
514
- is_marlin=True)
515
- config = get_config_func(M)
516
-
517
- block_size_m = config['BLOCK_SIZE_M']
518
-
519
- sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
520
-
521
- max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
522
- workspace = torch.zeros(max_workspace_size,
523
- dtype=torch.int,
524
- device="cuda",
525
- requires_grad=False)
526
-
527
- intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N),
528
- device=hidden_states.device,
529
- dtype=hidden_states.dtype)
530
-
531
- intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
532
- hidden_states, w1, sorted_token_ids, topk_weights, topk_ids, w1_scale,
533
- g_idx1, rand_perm1, workspace, M, 2 * N, K, True, E, topk,
534
- block_size_m, True, False)
535
-
536
- ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
537
-
538
- intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
539
- intermediate_cache2, w2, sorted_token_ids, topk_weights, topk_ids,
540
- w2_scale, g_idx2, rand_perm2, workspace, M, K, N, True, E, topk,
541
- block_size_m, False, True)
542
-
543
- return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
544
- dim=1)
545
-
546
-
449
def get_config_dtype_str(dtype: torch.dtype,
450
use_int8_w8a16: Optional[bool] = False,
451
use_fp8_w8a8: Optional[bool] = False):
vllm/model_executor/layers/fused_moe/layer.py
+51
-24
@@ -306,10 +306,28 @@ class FusedMoE(torch.nn.Module):
306
# Input scales can be loaded directly and should be equal.
307
param_data[expert_id] = loaded_weight
308
309
+ def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor,
310
+ shard_dim: int, loaded_weight: torch.tensor, tp_rank: int):
311
+
312
+ if shard_id == "w2":
313
+ self._load_w2(shard_id=shard_id,
314
+ shard_dim=shard_dim,
315
+ loaded_weight=loaded_weight,
316
+ expert_data=expert_data,
317
+ tp_rank=tp_rank)
318
+ else:
319
+ assert shard_id in ("w1", "w3")
320
+ expert_data.copy_(loaded_weight)
321
+
322
def weight_loader(self, param: torch.nn.Parameter,
323
loaded_weight: torch.Tensor, weight_name: str,
324
shard_id: str, expert_id: int) -> None:
325
326
+ # compressed-tensors represents weights on disk which are flipped
327
+ loaded_weight = loaded_weight.t().contiguous() if (
328
+ self.quant_method.__class__.__name__
329
+ == "CompressedTensorsMoEMethod") else loaded_weight
330
+
331
if shard_id not in ("w1", "w2", "w3"):
332
raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
333
f"got {shard_id}.")
@@ -325,19 +343,41 @@ class FusedMoE(torch.nn.Module):
343
expert_data = param.data[expert_id]
344
tp_rank = get_tensor_model_parallel_rank()
345
328
- # is_transposed: whether or not the parameter is transposed on disk
329
- # If transposed, the loaded weight will be transposed and the dim
330
- # to shard the loaded weight will be flipped.
346
+ # is_transposed: if the dim to shard the weight
347
+ # should be flipped. Required by GPTQ, compressed-tensors
348
+ # should be whatever dimension intermediate_size is
349
is_transposed = getattr(param, "is_transposed", False)
350
shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
351
if is_transposed:
334
- loaded_weight = loaded_weight.t().contiguous()
352
shard_dim = ~shard_dim
353
337
- # Case weight_scales
338
- if "weight_scale" in weight_name:
339
- # load the weight scaling based on the quantization scheme
340
- # supported weight scales can be found in
354
+ # Case input scale: input_scale loading is only supported for fp8
355
+ if "input_scale" in weight_name:
356
+ if param.data[expert_id] != 1 and (param.data[expert_id] -
357
+ loaded_weight).abs() > 1e-5:
358
+ raise ValueError(
359
+ "input_scales of w1 and w3 of a layer "
360
+ f"must be equal. But got {param.data[expert_id]} "
361
+ f"vs. {loaded_weight}")
362
+
363
+ self._load_single_value(param=param,
364
+ loaded_weight=loaded_weight,
365
+ expert_id=expert_id)
366
+ return
367
+
368
+ # Case g_idx
369
+ if "g_idx" in weight_name:
370
+ self._load_g_idx(shard_dim=0,
371
+ shard_id=shard_id,
372
+ loaded_weight=loaded_weight,
373
+ expert_data=expert_data,
374
+ tp_rank=tp_rank)
375
+ return
376
+
377
+ # Case weight scales and zero_points
378
+ if ("scale" in weight_name or "zero" in weight_name):
379
+ # load the weight scales and zp based on the quantization scheme
380
+ # supported weight scales/zp can be found in
381
# FusedMoeWeightScaleSupported
382
# TODO @dsikka: once hardened, refactor to use vLLM Parameters
383
# specific to each case
@@ -366,22 +406,9 @@ class FusedMoE(torch.nn.Module):
406
f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
407
return
408
409
+ # Case weight_shape
410
if "weight_shape" in weight_name:
370
- self._load_single_value(param=param,
371
- loaded_weight=loaded_weight,
372
- expert_id=expert_id)
373
- return
374
-
375
- # Case input scale
376
- if "input_scale" in weight_name:
377
- # Note: input_scale loading is only supported for fp8
378
- if param.data[expert_id] != 1 and (param.data[expert_id] -
379
- loaded_weight).abs() > 1e-5:
380
- raise ValueError(
381
- "input_scales of w1 and w3 of a layer "
382
- f"must be equal. But got {param.data[expert_id]} "
383
- f"vs. {loaded_weight}")
384
-
411
+ # only required by compressed-tensors
412
self._load_single_value(param=param,
413
loaded_weight=loaded_weight,
414
expert_id=expert_id)
@@ -498,4 +525,4 @@ class FusedMoE(torch.nn.Module):
525
param_data[expert_id][idx] = loaded_weight
526
# If we are in the row parallel case (down_proj)
527
else:
501
- param_data[expert_id] = loaded_weight
\ No newline at end of file
528
+ param_data[expert_id] = loaded_weight
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+28
-20
@@ -5,9 +5,7 @@ from typing import Callable, List, Optional
5
import torch
6
7
from vllm import _custom_ops as ops
8
-from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
9
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
- WNA16_SUPPORTED_BITS)
8
+from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
9
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
10
CompressionFormat)
11
from vllm.model_executor.utils import set_weight_attrs
@@ -40,11 +38,10 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
38
39
if not (self.quant_config.quant_format
40
== CompressionFormat.pack_quantized.value
43
- and self.num_bits in WNA16_SUPPORTED_BITS):
41
+ and self.num_bits == 4):
42
raise ValueError("For Fused MoE layers, only ",
43
f"{CompressionFormat.pack_quantized.value} ",
46
- "is supported for the following bits: ",
47
- f"{WNA16_SUPPORTED_BITS}")
44
+ "is supported for 4 bits")
45
46
def create_weights(self, layer: torch.nn.Module, num_experts: int,
47
hidden_size: int, intermediate_size: int,
@@ -269,19 +266,30 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
266
custom_routing_function: Optional[Callable] = None,
267
) -> torch.Tensor:
268
272
- from vllm.model_executor.layers.fused_moe.fused_moe import (
269
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
270
fused_marlin_moe)
271
275
- return fused_marlin_moe(x,
276
- layer.w13_weight_packed,
277
- layer.w2_weight_packed,
278
- router_logits,
279
- layer.w13_g_idx,
280
- layer.w2_g_idx,
281
- layer.w13_g_idx_sort_indices,
282
- layer.w2_g_idx_sort_indices,
283
- top_k,
284
- custom_routing_function,
285
- renormalize=renormalize,
286
- w1_scale=layer.w13_weight_scale,
287
- w2_scale=layer.w2_weight_scale)
272
+ topk_weights, topk_ids = FusedMoE.select_experts(
273
+ hidden_states=x,
274
+ router_logits=router_logits,
275
+ use_grouped_topk=use_grouped_topk,
276
+ top_k=top_k,
277
+ renormalize=renormalize,
278
+ topk_group=topk_group,
279
+ num_expert_group=num_expert_group,
280
+ custom_routing_function=custom_routing_function)
281
+
282
+ return fused_marlin_moe(
283
+ x,
284
+ layer.w13_weight_packed,
285
+ layer.w2_weight_packed,
286
+ router_logits,
287
+ layer.w13_g_idx,
288
+ layer.w2_g_idx,
289
+ layer.w13_g_idx_sort_indices,
290
+ layer.w2_g_idx_sort_indices,
291
+ topk_weights,
292
+ topk_ids,
293
+ w1_scale=layer.w13_weight_scale,
294
+ w2_scale=layer.w2_weight_scale,
295
+ )
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.scalar_type import scalar_types
22
__all__ = ["CompressedTensorsWNA16"]
23
WNA16_SUPPORTED_TYPES_MAP = {
24
4: scalar_types.uint4b8,
25
- 8: scalar_types.uint8b128,
25
+ 8: scalar_types.uint8b128
26
}
27
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
28
vllm/model_executor/layers/quantization/gptq_marlin.py
+297
-15
@@ -1,18 +1,22 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Callable, Dict, List, Optional, Union
2
3
import torch
4
from torch.nn import Parameter
5
6
from vllm import _custom_ops as ops
7
from vllm.logger import init_logger
8
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
+from vllm.model_executor.layers.fused_moe.layer import (
9
+ FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
+ set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
apply_gptq_marlin_linear, check_marlin_supported, marlin_is_k_full,
13
- marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
14
- marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
15
- verify_marlin_supported, verify_marlin_supports_shape)
16
+ marlin_make_empty_g_idx, marlin_make_workspace, marlin_moe_permute_scales,
17
+ marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
18
+ marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
19
+ verify_marlin_supports_shape)
20
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
22
GroupQuantScaleParameter,
@@ -33,8 +37,14 @@ class GPTQMarlinConfig(QuantizationConfig):
37
(8, True): scalar_types.uint8b128,
38
}
39
36
- def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
37
- is_sym: bool, lm_head_quantized: bool) -> None:
40
+ def __init__(
41
+ self,
42
+ weight_bits: int,
43
+ group_size: int,
44
+ desc_act: bool,
45
+ is_sym: bool,
46
+ lm_head_quantized: bool,
47
+ ) -> None:
48
if desc_act and group_size == -1:
49
# In this case, act_order == True is the same as act_order == False
50
# (since we have only one group per output channel)
@@ -105,11 +115,14 @@ class GPTQMarlinConfig(QuantizationConfig):
115
" faster inference")
116
return None
117
108
- def get_quant_method(self, layer: torch.nn.Module,
109
- prefix: str) -> Optional["GPTQMarlinLinearMethod"]:
110
- if (isinstance(layer, LinearBase) or
111
- (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
118
+ def get_quant_method(
119
+ self, layer: torch.nn.Module, prefix: str
120
+ ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod"]]:
121
+ if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead)
122
+ and self.lm_head_quantized):
123
return GPTQMarlinLinearMethod(self)
124
+ elif isinstance(layer, FusedMoE):
125
+ return GPTQMarlinMoEMethod(self)
126
return None
127
128
def get_scaled_act_names(self) -> List[str]:
@@ -179,7 +192,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
192
output_size_per_partition=output_size_per_partition,
193
input_size_per_partition=input_size_per_partition,
194
input_size=input_size,
182
- group_size=group_size)
195
+ group_size=group_size,
196
+ )
197
198
# Determine sharding
199
if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
@@ -299,7 +313,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
313
perm=layer.g_idx_sort_indices,
314
size_k=layer.input_size_per_partition,
315
size_n=layer.output_size_per_partition,
302
- num_bits=self.quant_config.quant_type.size_bits)
316
+ num_bits=self.quant_config.quant_type.size_bits,
317
+ )
318
replace_tensor(layer, "qweight", marlin_qweight)
319
320
# Permute scales from autogptq format to marlin format.
@@ -308,7 +323,8 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
323
size_k=(layer.input_size if self.quant_config.desc_act else
324
layer.input_size_per_partition),
325
size_n=layer.output_size_per_partition,
311
- group_size=self.quant_config.group_size)
326
+ group_size=self.quant_config.group_size,
327
+ )
328
replace_tensor(layer, "scales", marlin_scales)
329
330
def apply(
@@ -329,4 +345,270 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
345
output_size_per_partition=layer.output_size_per_partition,
346
input_size_per_partition=layer.input_size_per_partition,
347
is_k_full=layer.is_k_full,
332
- bias=bias)
348
+ bias=bias,
349
+ )
350
+
351
+
352
+class GPTQMarlinMoEMethod(FusedMoEMethodBase):
353
+ """MoE Marlin method with quantization."""
354
+
355
+ def __init__(self, quant_config: GPTQMarlinConfig) -> None:
356
+ self.quant_config = quant_config
357
+
358
+ def create_weights(
359
+ self,
360
+ layer: torch.nn.Module,
361
+ num_experts: int,
362
+ hidden_size: int,
363
+ intermediate_size: int,
364
+ params_dtype: torch.dtype,
365
+ **extra_weight_attrs,
366
+ ):
367
+ # Currently assuming is_k_full is always True
368
+ # (input size per partition is the same as full input size)
369
+ # Supports only sym for now (no zp)
370
+ if self.quant_config.group_size != -1:
371
+ scales_size13 = hidden_size // self.quant_config.group_size
372
+ scales_size2 = intermediate_size // self.quant_config.group_size
373
+ strategy = FusedMoeWeightScaleSupported.GROUP.value
374
+ else:
375
+ scales_size13 = 1
376
+ scales_size2 = 1
377
+ strategy = FusedMoeWeightScaleSupported.CHANNEL.value
378
+
379
+ extra_weight_attrs.update({
380
+ "quant_method": strategy,
381
+ "is_transposed": True
382
+ })
383
+ # Fused gate_up_proj (column parallel)
384
+ w13_qweight = torch.nn.Parameter(
385
+ torch.empty(
386
+ num_experts,
387
+ hidden_size // self.quant_config.pack_factor,
388
+ 2 * intermediate_size,
389
+ dtype=torch.int32,
390
+ ),
391
+ requires_grad=False,
392
+ )
393
+ layer.register_parameter("w13_qweight", w13_qweight)
394
+ set_weight_attrs(w13_qweight, extra_weight_attrs)
395
+ # down_proj (row parallel)
396
+ w2_qweight = torch.nn.Parameter(
397
+ torch.empty(
398
+ num_experts,
399
+ intermediate_size // self.quant_config.pack_factor,
400
+ hidden_size,
401
+ dtype=torch.int32,
402
+ ),
403
+ requires_grad=False,
404
+ )
405
+ layer.register_parameter("w2_qweight", w2_qweight)
406
+ set_weight_attrs(w2_qweight, extra_weight_attrs)
407
+ # up_proj scales
408
+ w13_scales = torch.nn.Parameter(
409
+ torch.empty(num_experts,
410
+ scales_size13,
411
+ 2 * intermediate_size,
412
+ dtype=torch.half),
413
+ requires_grad=False,
414
+ )
415
+ layer.register_parameter("w13_scales", w13_scales)
416
+ set_weight_attrs(w13_scales, extra_weight_attrs)
417
+ # down_proj scales
418
+ w2_scales = torch.nn.Parameter(
419
+ torch.empty(num_experts,
420
+ scales_size2,
421
+ hidden_size,
422
+ dtype=torch.half),
423
+ requires_grad=False,
424
+ )
425
+ layer.register_parameter("w2_scales", w2_scales)
426
+ set_weight_attrs(w2_scales, extra_weight_attrs)
427
+ # up_proj scales
428
+ w13_qzeros = torch.nn.Parameter(
429
+ torch.empty(num_experts,
430
+ scales_size13,
431
+ 2 * intermediate_size // self.quant_config.pack_factor,
432
+ dtype=params_dtype),
433
+ requires_grad=False,
434
+ )
435
+ layer.register_parameter("w13_qzeros", w13_qzeros)
436
+ set_weight_attrs(w13_qzeros, extra_weight_attrs)
437
+ # down_proj scales
438
+ w2_qzeros = torch.nn.Parameter(
439
+ torch.empty(num_experts,
440
+ scales_size2,
441
+ hidden_size // self.quant_config.pack_factor,
442
+ dtype=params_dtype),
443
+ requires_grad=False,
444
+ )
445
+ layer.register_parameter("w2_qzeros", w2_qzeros)
446
+ set_weight_attrs(w2_qzeros, extra_weight_attrs)
447
+ w13_g_idx = torch.nn.Parameter(
448
+ torch.empty(
449
+ num_experts,
450
+ hidden_size,
451
+ dtype=torch.int32,
452
+ ),
453
+ requires_grad=False,
454
+ )
455
+ layer.register_parameter("w13_g_idx", w13_g_idx)
456
+ set_weight_attrs(w13_g_idx, extra_weight_attrs)
457
+ w2_g_idx = torch.nn.Parameter(
458
+ torch.empty(
459
+ num_experts,
460
+ intermediate_size,
461
+ dtype=torch.int32,
462
+ ),
463
+ requires_grad=False,
464
+ )
465
+ layer.register_parameter("w2_g_idx", w2_g_idx)
466
+ set_weight_attrs(w2_g_idx, extra_weight_attrs)
467
+ w13_g_idx_sort_indices = torch.nn.Parameter(
468
+ torch.empty(
469
+ num_experts,
470
+ hidden_size,
471
+ dtype=torch.int32,
472
+ ),
473
+ requires_grad=False,
474
+ )
475
+ layer.register_parameter("w13_g_idx_sort_indices",
476
+ w13_g_idx_sort_indices)
477
+ set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
478
+ w2_g_idx_sort_indices = torch.nn.Parameter(
479
+ torch.empty(
480
+ num_experts,
481
+ intermediate_size,
482
+ dtype=torch.int32,
483
+ ),
484
+ requires_grad=False,
485
+ )
486
+ layer.register_parameter("w2_g_idx_sort_indices",
487
+ w2_g_idx_sort_indices)
488
+ set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
489
+
490
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
491
+
492
+ # Process act_order
493
+ if self.quant_config.desc_act:
494
+ # Get sorting based on g_idx
495
+ num_experts = layer.w13_g_idx.shape[0]
496
+ w13_g_idx_sort_indices = torch.empty_like(layer.w13_g_idx)
497
+ w2_g_idx_sort_indices = torch.empty_like(layer.w2_g_idx)
498
+ w13_sorted_g_idx = torch.empty_like(layer.w13_g_idx)
499
+ w2_sorted_g_idx = torch.empty_like(layer.w2_g_idx)
500
+ for e in range(num_experts):
501
+ w13_g_idx_sort_indices[e] = torch.argsort(
502
+ layer.w13_g_idx[e]).to(torch.int32)
503
+ w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_g_idx[e]).to(
504
+ torch.int32)
505
+ w13_sorted_g_idx[e] = layer.w13_g_idx[e][
506
+ w13_g_idx_sort_indices[e]]
507
+ w2_sorted_g_idx[e] = layer.w2_g_idx[e][
508
+ w2_g_idx_sort_indices[e]]
509
+ replace_tensor(layer, "w13_g_idx", w13_sorted_g_idx)
510
+ replace_tensor(layer, "w2_g_idx", w2_sorted_g_idx)
511
+ replace_tensor(layer, "w13_g_idx_sort_indices",
512
+ w13_g_idx_sort_indices)
513
+ replace_tensor(layer, "w2_g_idx_sort_indices",
514
+ w2_g_idx_sort_indices)
515
+ else:
516
+ # Reset g_idx related tensors
517
+ num_experts = layer.w13_g_idx.shape[0]
518
+ device = layer.w13_g_idx.device
519
+ layer.w13_g_idx = torch.nn.Parameter(
520
+ torch.empty((num_experts, 0), dtype=torch.int32,
521
+ device=device),
522
+ requires_grad=False,
523
+ )
524
+ layer.w2_g_idx = torch.nn.Parameter(
525
+ torch.empty((num_experts, 0), dtype=torch.int32,
526
+ device=device),
527
+ requires_grad=False,
528
+ )
529
+ layer.w13_g_idx_sort_indices = torch.nn.Parameter(
530
+ torch.empty((num_experts, 0), dtype=torch.int32,
531
+ device=device),
532
+ requires_grad=False,
533
+ )
534
+ layer.w2_g_idx_sort_indices = torch.nn.Parameter(
535
+ torch.empty((num_experts, 0), dtype=torch.int32,
536
+ device=device),
537
+ requires_grad=False,
538
+ )
539
+ # Repack weights
540
+ marlin_w13_qweight = ops.gptq_marlin_moe_repack(
541
+ layer.w13_qweight,
542
+ layer.w13_g_idx_sort_indices,
543
+ layer.w13_qweight.shape[1] * self.quant_config.pack_factor,
544
+ layer.w13_qweight.shape[2],
545
+ self.quant_config.quant_type.size_bits,
546
+ )
547
+ replace_tensor(layer, "w13_qweight", marlin_w13_qweight)
548
+ marlin_w2_qweight = ops.gptq_marlin_moe_repack(
549
+ layer.w2_qweight,
550
+ layer.w2_g_idx_sort_indices,
551
+ layer.w2_qweight.shape[1] * self.quant_config.pack_factor,
552
+ layer.w2_qweight.shape[2],
553
+ self.quant_config.quant_type.size_bits,
554
+ )
555
+ replace_tensor(layer, "w2_qweight", marlin_w2_qweight)
556
+ # Repack scales
557
+ marlin_w13_scales = marlin_moe_permute_scales(
558
+ s=layer.w13_scales,
559
+ size_k=layer.intermediate_size_per_partition,
560
+ size_n=layer.w13_scales.shape[2],
561
+ group_size=self.quant_config.group_size,
562
+ )
563
+ replace_tensor(layer, "w13_scales", marlin_w13_scales)
564
+ marlin_w2_scales = marlin_moe_permute_scales(
565
+ s=layer.w2_scales,
566
+ size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor,
567
+ size_n=layer.w2_scales.shape[2],
568
+ group_size=self.quant_config.group_size,
569
+ )
570
+ replace_tensor(layer, "w2_scales", marlin_w2_scales)
571
+
572
+ def apply(
573
+ self,
574
+ layer: torch.nn.Module,
575
+ x: torch.Tensor,
576
+ router_logits: torch.Tensor,
577
+ top_k: int,
578
+ renormalize: bool = True,
579
+ use_grouped_topk: bool = False,
580
+ num_expert_group: Optional[int] = None,
581
+ topk_group: Optional[int] = None,
582
+ custom_routing_function: Optional[Callable] = None,
583
+ ) -> torch.Tensor:
584
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
585
+ fused_marlin_moe)
586
+
587
+ # The input must currently be float16
588
+ orig_dtype = x.dtype
589
+ x = x.half()
590
+
591
+ topk_weights, topk_ids = FusedMoE.select_experts(
592
+ hidden_states=x,
593
+ router_logits=router_logits,
594
+ use_grouped_topk=use_grouped_topk,
595
+ top_k=top_k,
596
+ renormalize=renormalize,
597
+ topk_group=topk_group,
598
+ num_expert_group=num_expert_group,
599
+ custom_routing_function=None)
600
+
601
+ return fused_marlin_moe(
602
+ x,
603
+ layer.w13_qweight,
604
+ layer.w2_qweight,
605
+ router_logits,
606
+ layer.w13_g_idx,
607
+ layer.w2_g_idx,
608
+ layer.w13_g_idx_sort_indices,
609
+ layer.w2_g_idx_sort_indices,
610
+ topk_weights,
611
+ topk_ids,
612
+ w1_scale=layer.w13_scales,
613
+ w2_scale=layer.w2_scales,
614
+ ).to(orig_dtype)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+17
@@ -176,6 +176,23 @@ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
176
return s
177
178
179
+def marlin_moe_permute_scales(
180
+ s: torch.Tensor,
181
+ size_k: int,
182
+ size_n: int,
183
+ group_size: int,
184
+):
185
+ num_experts = s.shape[0]
186
+ output = torch.empty(
187
+ (num_experts, s.shape[1], s.shape[2]),
188
+ device=s.device,
189
+ dtype=s.dtype,
190
+ )
191
+ for e in range(num_experts):
192
+ output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size)
193
+ return output
194
+
195
+
196
def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
197
num_bits: int) -> torch.Tensor:
198
# Permute zero-points in a similar way to scales, but do not use the
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
+7
-4
@@ -1,6 +1,6 @@
1
"""Utility functions used for tests and benchmarks"""
2
3
-from typing import List
3
+from typing import List, Optional
4
5
import numpy as np
6
import torch
@@ -92,8 +92,11 @@ def get_weight_perm(num_bits: int):
92
return perm
93
94
95
-def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
96
- act_order: bool):
95
+def marlin_quantize(w: torch.Tensor,
96
+ quant_type: ScalarType,
97
+ group_size: int,
98
+ act_order: bool,
99
+ test_perm: Optional[torch.Tensor] = None):
100
size_k, size_n = w.shape
101
num_bits = quant_type.size_bits
102
@@ -104,7 +107,7 @@ def marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int,
107
108
# Quantize (and apply act_order if provided)
109
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
107
- w, quant_type, group_size, act_order)
110
+ w, quant_type, group_size, act_order, test_perm)
111
112
# For act_order, sort the "weights" and "g_idx" so that group ids are
113
# increasing
vllm/model_executor/layers/quantization/utils/quant_utils.py
+13
-6
@@ -1,5 +1,5 @@
1
"""This file is used for /tests and /benchmarks"""
2
-from typing import List
2
+from typing import List, Optional
3
4
import numpy
5
import torch
@@ -53,7 +53,10 @@ def get_pack_factor(num_bits):
53
return 32 // num_bits
54
55
56
-def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
56
+def permute_rows(q_w: torch.Tensor,
57
+ w_ref: torch.Tensor,
58
+ group_size: int,
59
+ test_perm: Optional[torch.Tensor] = None):
60
assert q_w.shape == w_ref.shape
61
62
orig_device = q_w.device
@@ -64,7 +67,7 @@ def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
67
g_idx[i] = i // group_size
68
69
# Simulate act_order by doing a random permutation on K
67
- rand_perm = torch.randperm(k_size)
70
+ rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
71
72
g_idx = g_idx[rand_perm].contiguous()
73
q_w = q_w[rand_perm, :].contiguous()
@@ -164,8 +167,11 @@ def quantize_weights(w: torch.Tensor,
167
)
168
169
167
-def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
168
- group_size: int, act_order: bool):
170
+def gptq_quantize_weights(w: torch.Tensor,
171
+ quant_type: ScalarType,
172
+ group_size: int,
173
+ act_order: bool,
174
+ test_perm: Optional[torch.Tensor] = None):
175
size_k, _ = w.shape
176
177
assert w.is_floating_point(), "w must be float"
@@ -186,7 +192,8 @@ def gptq_quantize_weights(w: torch.Tensor, quant_type: ScalarType,
192
), "For act_order, groupsize = {} must be less than size_k = {}".format(
193
group_size, size_k)
194
189
- w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size)
195
+ w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size,
196
+ test_perm)
197
198
return w_ref, w_q, w_s, g_idx, rand_perm
199
vllm/model_executor/model_loader/utils.py
+8
@@ -24,10 +24,18 @@ def get_model_architecture(
24
# Special handling for quantized Mixtral.
25
# FIXME(woosuk): This is a temporary hack.
26
mixtral_supported = ["fp8", "compressed-tensors"]
27
+ # for gptq_marlin, only run fused MoE for int4
28
+ if model_config.quantization == "gptq_marlin":
29
+ hf_quant_config = getattr(model_config.hf_config,
30
+ "quantization_config", None)
31
+ if hf_quant_config and hf_quant_config.get("bits") == 4:
32
+ mixtral_supported.append("gptq_marlin")
33
+
34
if (model_config.quantization is not None
35
and model_config.quantization not in mixtral_supported
36
and "MixtralForCausalLM" in architectures):
37
architectures = ["QuantMixtralForCausalLM"]
38
+
39
return ModelRegistry.resolve_model_cls(architectures)
40
41
vllm/model_executor/models/mixtral.py
+7
-2
@@ -435,7 +435,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
435
continue
436
name = name.replace(weight_name, param_name)
437
# Skip loading extra bias for GPTQ models.
438
- if name.endswith(".bias") and name not in params_dict:
438
+ if ((name.endswith(".bias") or name.endswith("_bias"))
439
+ and name not in params_dict):
440
continue
441
# Skip layers on other devices.
442
if is_pp_missing_parameter(name, self):
@@ -454,6 +455,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
455
# Skip layers on other devices.
456
if is_pp_missing_parameter(name, self):
457
continue
458
+ if ((name.endswith(".bias") or name.endswith("_bias"))
459
+ and name not in params_dict):
460
+ continue
461
param = params_dict[name]
462
weight_loader = param.weight_loader
463
weight_loader(param,
@@ -464,7 +468,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
468
break
469
else:
470
# Skip loading extra bias for GPTQ models.
467
- if name.endswith(".bias") and name not in params_dict:
471
+ if ((name.endswith(".bias") or name.endswith("_bias"))
472
+ and name not in params_dict):
473
continue
474
# Skip layers on other devices.
475
if is_pp_missing_parameter(name, self):