A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel] add triton fused moe kernel for gptq/awq (#12185)
Jinzhen Lin committed
Jan 29, 2025 at 22:07 UTC
27b78c73cad00f5c7bb3b2431f02dc680f7034bc
4 files changed
+874
-55
tests/kernels/test_moe.py
+91
@@ -18,6 +18,8 @@ from vllm.model_executor.layers.fused_moe.moe_torch_iterative import (
18
fused_moe as iterative_moe)
19
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20
marlin_quantize)
21
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
22
+ quantize_weights)
23
from vllm.model_executor.models.mixtral import MixtralMoE
24
from vllm.platforms import current_platform
25
from vllm.scalar_type import scalar_types
@@ -55,6 +57,95 @@ def test_fused_moe(
57
rtol=0)
58
59
60
+@pytest.mark.parametrize("m", [1, 32, 222])
61
+@pytest.mark.parametrize("n", [128, 1024, 2048])
62
+@pytest.mark.parametrize("k", [128, 1024])
63
+@pytest.mark.parametrize("e", NUM_EXPERTS)
64
+@pytest.mark.parametrize("topk", TOP_KS)
65
+@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
66
+@pytest.mark.parametrize("group_size", [64, 128])
67
+@pytest.mark.parametrize("has_zp", [True, False])
68
+@pytest.mark.parametrize("weight_bits", [4, 8])
69
+def test_fused_moe_wn16(m: int, n: int, k: int, e: int, topk: int,
70
+ dtype: torch.dtype, group_size: int, has_zp: bool,
71
+ weight_bits: int):
72
+ print(m, n, k, e, topk, dtype, group_size, has_zp, weight_bits)
73
+ a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
74
+ w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
75
+ w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
76
+ score = torch.randn((m, e), device="cuda", dtype=dtype)
77
+
78
+ if weight_bits == 4:
79
+ pack_factor = 2
80
+ quant_type = scalar_types.uint4 if has_zp else scalar_types.uint4b8
81
+ elif weight_bits == 8:
82
+ pack_factor = 1
83
+ quant_type = scalar_types.uint8 if has_zp else scalar_types.uint8b128
84
+
85
+ w1_ref = w1.clone()
86
+ w2_ref = w2.clone()
87
+ w1_qweight = torch.empty((e, 2 * n, k // pack_factor),
88
+ device="cuda",
89
+ dtype=torch.uint8)
90
+ w2_qweight = torch.empty((e, k, n // pack_factor),
91
+ device="cuda",
92
+ dtype=torch.uint8)
93
+ w1_scales = torch.empty((e, 2 * n, k // group_size),
94
+ device="cuda",
95
+ dtype=dtype)
96
+ w2_scales = torch.empty((e, k, n // group_size),
97
+ device="cuda",
98
+ dtype=dtype)
99
+ w1_qzeros = torch.empty((e, 2 * n // pack_factor, k // group_size),
100
+ device="cuda",
101
+ dtype=torch.uint8)
102
+ w2_qzeros = torch.empty((e, k // pack_factor, n // group_size),
103
+ device="cuda",
104
+ dtype=torch.uint8)
105
+
106
+ for i in range(e * 2):
107
+ expert_id = i % e
108
+ if i // e == 0:
109
+ w, w_ref, w_qweight, w_scales, w_qzeros = \
110
+ w1, w1_ref, w1_qweight, w1_scales, w1_qzeros
111
+ else:
112
+ w, w_ref, w_qweight, w_scales, w_qzeros = \
113
+ w2, w2_ref, w2_qweight, w2_scales, w2_qzeros
114
+ weight, qweight, scales, qzeros = quantize_weights(
115
+ w[expert_id].T, quant_type, group_size, has_zp, False)
116
+ weight = weight.T
117
+ qweight = qweight.T.contiguous().to(torch.uint8)
118
+ scales = scales.T
119
+ if has_zp:
120
+ qzeros = qzeros.T.contiguous().to(torch.uint8)
121
+ if weight_bits == 4:
122
+ qweight = qweight[:, 1::2] * 16 + qweight[:, ::2]
123
+ if has_zp:
124
+ qzeros = qzeros[1::2, :] * 16 + qzeros[::2, :]
125
+
126
+ w_ref[expert_id] = weight
127
+ w_qweight[expert_id] = qweight
128
+ w_scales[expert_id] = scales
129
+ if has_zp:
130
+ w_qzeros[expert_id] = qzeros
131
+
132
+ triton_output = fused_moe(a,
133
+ w1_qweight,
134
+ w2_qweight,
135
+ score,
136
+ topk,
137
+ renormalize=False,
138
+ use_int4_w4a16=weight_bits == 4,
139
+ use_int8_w8a16=weight_bits == 8,
140
+ w1_scale=w1_scales,
141
+ w2_scale=w2_scales,
142
+ w1_zp=w1_qzeros if has_zp else None,
143
+ w2_zp=w2_qzeros if has_zp else None,
144
+ block_shape=[0, group_size])
145
+ torch_output = torch_moe(a, w1_ref, w2_ref, score, topk)
146
+ torch.testing.assert_close(triton_output, torch_output, atol=2e-2, rtol=0)
147
+
148
+
149
@pytest.mark.parametrize("dtype",
150
[torch.float32, torch.float16, torch.bfloat16])
151
@torch.inference_mode()
vllm/model_executor/layers/fused_moe/fused_moe.py
+354
-53
@@ -19,6 +19,206 @@ from vllm.utils import direct_register_custom_op
19
logger = init_logger(__name__)
20
21
22
+@triton.jit
23
+def fused_moe_kernel_gptq_awq(
24
+ # Pointers to matrices
25
+ a_ptr,
26
+ b_ptr,
27
+ c_ptr,
28
+ b_scale_ptr,
29
+ b_zp_ptr,
30
+ topk_weights_ptr,
31
+ sorted_token_ids_ptr,
32
+ expert_ids_ptr,
33
+ num_tokens_post_padded_ptr,
34
+ # Matrix dimensions
35
+ N: tl.constexpr,
36
+ K: tl.constexpr,
37
+ EM,
38
+ num_valid_tokens,
39
+ # The stride variables represent how much to increase the ptr by when
40
+ # moving by 1 element in a particular dimension. E.g. `stride_am` is
41
+ # how much to increase `a_ptr` by to get the element one row down
42
+ # (A has M rows).
43
+ stride_am,
44
+ stride_ak,
45
+ stride_be,
46
+ stride_bk,
47
+ stride_bn,
48
+ stride_cm,
49
+ stride_cn,
50
+ stride_bse,
51
+ stride_bsk,
52
+ stride_bsn,
53
+ stride_bze,
54
+ stride_bzk,
55
+ stride_bzn,
56
+ block_k_diviable: tl.constexpr,
57
+ group_size: tl.constexpr,
58
+ # Meta-parameters
59
+ BLOCK_SIZE_M: tl.constexpr,
60
+ BLOCK_SIZE_N: tl.constexpr,
61
+ BLOCK_SIZE_K: tl.constexpr,
62
+ GROUP_SIZE_M: tl.constexpr,
63
+ MUL_ROUTED_WEIGHT: tl.constexpr,
64
+ top_k: tl.constexpr,
65
+ compute_type: tl.constexpr,
66
+ has_zp: tl.constexpr,
67
+ use_int4_w4a16: tl.constexpr,
68
+ use_int8_w8a16: tl.constexpr):
69
+ """
70
+ Implements the fused computation for a Mixture of Experts (MOE) using
71
+ token and expert matrices.
72
+
73
+ Key Parameters:
74
+ - A: The input tensor representing tokens with shape (*, K), where '*' can
75
+ be any shape representing batches and K is the feature dimension of
76
+ each token.
77
+ - B: The stacked MOE weight tensor with shape (E, N, K), where E is
78
+ the number of experts, K is the input feature dimension, and N is
79
+ the output feature dimension.
80
+ - C: The output cache tensor with shape (M, topk, N), where M is the
81
+ total number of tokens post padding, topk is the number of times
82
+ each token is repeated, and N is the output feature dimension.
83
+ - sorted_token_ids: A tensor containing the sorted indices of tokens,
84
+ repeated topk times and arranged by the expert index they are
85
+ assigned to.
86
+ - expert_ids: A tensor containing the indices of the expert for each
87
+ block. It determines which expert matrix from B should be used for
88
+ each block in A.
89
+ This kernel performs the multiplication of a token by its corresponding
90
+ expert matrix as determined by `expert_ids`. The sorting of
91
+ `sorted_token_ids` by expert index and padding ensures divisibility by
92
+ BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix
93
+ multiplication across different blocks processed by the same expert.
94
+ """
95
+ # -----------------------------------------------------------
96
+ # Map program ids `pid` to the block of C it should compute.
97
+ # This is done in a grouped ordering to promote L2 data reuse.
98
+ pid = tl.program_id(axis=0)
99
+ num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
100
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
101
+ num_pid_in_group = GROUP_SIZE_M * num_pid_n
102
+ group_id = pid // num_pid_in_group
103
+ first_pid_m = group_id * GROUP_SIZE_M
104
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
105
+ pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
106
+ pid_n = (pid % num_pid_in_group) // group_size_m
107
+
108
+ # ----------------------------------------------------------
109
+ # Create pointers for the first blocks of A and B.
110
+ # We will advance this pointer as we move in the K direction
111
+ # and accumulate
112
+ # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers
113
+ # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers
114
+ num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
115
+ if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
116
+ return
117
+ offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(
118
+ tl.int64)
119
+ offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
120
+ token_mask = offs_token < num_valid_tokens
121
+
122
+ offs_bn = (pid_n * BLOCK_SIZE_N +
123
+ tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
124
+ offs_k = tl.arange(0, BLOCK_SIZE_K)
125
+ a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
126
+ offs_k[None, :] * stride_ak)
127
+
128
+ off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
129
+
130
+ if use_int4_w4a16:
131
+ b_ptrs = b_ptr + off_experts * stride_be + \
132
+ (offs_k[:, None] // 2) * stride_bk + offs_bn[None, :] * stride_bn
133
+ b_shifter = (offs_k[:, None] % 2) * 4
134
+ elif use_int8_w8a16:
135
+ b_ptrs = b_ptr + off_experts * stride_be + \
136
+ offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn
137
+
138
+ if not has_zp and use_int4_w4a16:
139
+ b_zp_num = 8
140
+ if not has_zp and use_int8_w8a16:
141
+ b_zp_num = 128
142
+ elif has_zp and use_int4_w4a16:
143
+ b_zp_shifter = (offs_bn[None, :] % 2) * 4
144
+
145
+ # -----------------------------------------------------------
146
+ # Iterate to compute a block of the C matrix.
147
+ # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
148
+ # of fp32 values for higher accuracy.
149
+ # `accumulator` will be converted back to fp16 after the loop.
150
+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
151
+ for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
152
+ # Load the next block of A and B, generate a mask by checking the
153
+ # K dimension.
154
+
155
+ if not block_k_diviable:
156
+ k_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K
157
+ k_other = 0.0
158
+ else:
159
+ k_mask = None
160
+ k_other = None
161
+
162
+ a = tl.load(a_ptrs,
163
+ mask=token_mask[:, None] &
164
+ (offs_k[None, :] < K - k * BLOCK_SIZE_K),
165
+ other=0.0)
166
+ b = tl.load(b_ptrs)
167
+ if use_int4_w4a16:
168
+ b = (b >> b_shifter) & 0xF
169
+
170
+ b_scale_ptrs = b_scale_ptr + off_experts * stride_bse + \
171
+ offs_bn[None, :] * stride_bsn + \
172
+ ((offs_k[:, None] + BLOCK_SIZE_K * k) // group_size) * stride_bsk
173
+ b_scale = tl.load(b_scale_ptrs, mask=k_mask, other=k_other)
174
+ b_scale = b_scale.to(tl.float32)
175
+
176
+ if has_zp and use_int4_w4a16:
177
+ offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size
178
+ b_zp_ptrs = b_zp_ptr + off_experts * stride_bze + \
179
+ (offs_bn[None, :] // 2) * stride_bzn + \
180
+ offs_k_true * stride_bzk
181
+ b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other)
182
+ b_zp = ((b_zp >> b_zp_shifter) & 0xF)
183
+ b_zp = b_zp.to(tl.float32)
184
+ elif has_zp and use_int8_w8a16:
185
+ offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size
186
+ b_zp_ptrs = b_zp_ptr + off_experts * stride_bze + \
187
+ offs_bn[None, :] * stride_bzn + \
188
+ offs_k_true * stride_bzk
189
+ b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other)
190
+ b_zp = b_zp.to(tl.float32)
191
+
192
+ # We accumulate along the K dimension.
193
+ if has_zp:
194
+ b = ((b.to(tl.float32) - b_zp) * b_scale).to(compute_type)
195
+ else:
196
+ b = ((b.to(tl.float32) - b_zp_num) * b_scale).to(compute_type)
197
+ accumulator = tl.dot(a, b, acc=accumulator)
198
+
199
+ # Advance the ptrs to the next K block.
200
+ a_ptrs += BLOCK_SIZE_K * stride_ak
201
+ if use_int4_w4a16:
202
+ b_ptrs += (BLOCK_SIZE_K // 2) * stride_bk
203
+ else:
204
+ b_ptrs += BLOCK_SIZE_K * stride_bk
205
+
206
+ if MUL_ROUTED_WEIGHT:
207
+ moe_weight = tl.load(topk_weights_ptr + offs_token,
208
+ mask=token_mask,
209
+ other=0)
210
+ accumulator = accumulator * moe_weight[:, None]
211
+
212
+ accumulator = accumulator.to(compute_type)
213
+ # -----------------------------------------------------------
214
+ # Write back the block of the output
215
+ offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
216
+ c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[
217
+ None, :]
218
+ c_mask = token_mask[:, None] & (offs_cn[None, :] < N)
219
+ tl.store(c_ptrs, accumulator, mask=c_mask)
220
+
221
+
222
@triton.jit
223
def fused_moe_kernel(
224
# Pointers to matrices
@@ -266,6 +466,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
466
C: torch.Tensor,
467
A_scale: Optional[torch.Tensor],
468
B_scale: Optional[torch.Tensor],
469
+ B_zp: Optional[torch.Tensor],
470
topk_weights: torch.Tensor,
471
topk_ids: torch.Tensor,
472
sorted_token_ids: torch.Tensor,
@@ -277,6 +478,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
478
compute_type: tl.dtype,
479
use_fp8_w8a8: bool,
480
use_int8_w8a16: bool,
481
+ use_int4_w4a16: bool,
482
block_shape: Optional[List[int]] = None) -> None:
483
assert topk_weights.stride(1) == 1
484
assert sorted_token_ids.stride(0) == 1
@@ -292,50 +494,108 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
494
assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1]
495
assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2]
496
assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1]
295
- elif use_int8_w8a16:
497
+ elif use_int8_w8a16 or use_int4_w4a16:
498
assert B_scale is not None
499
+ assert block_shape is None or block_shape[0] == 0
500
else:
501
assert A_scale is None
502
assert B_scale is None
503
301
- grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
302
- 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
504
+ EM = sorted_token_ids.shape[0]
505
+ if A.shape[0] < config["BLOCK_SIZE_M"]:
506
+ # optimize for small batch_size.
507
+ # We assume that top_ids of each token is unique, so
508
+ # so num_valid_experts <= batch_size <= BLOCK_SIZE_M,
509
+ # and we can skip some invalid blocks.
510
+ EM = min(sorted_token_ids.shape[0],
511
+ A.shape[0] * top_k * config['BLOCK_SIZE_M'])
512
+ grid = lambda META: (triton.cdiv(EM, META['BLOCK_SIZE_M']) * triton.cdiv(
513
+ B.shape[1], META['BLOCK_SIZE_N']), )
514
+
515
+ if (use_int8_w8a16 or use_int4_w4a16) and \
516
+ block_shape is not None and block_shape[1] > 0:
517
+ assert B_scale is not None and B_scale.ndim == 3
518
+ assert B_zp is None or B_zp.ndim == 3
519
+
520
+ fused_moe_kernel_gptq_awq[grid](
521
+ A,
522
+ B,
523
+ C,
524
+ B_scale,
525
+ B_zp,
526
+ topk_weights,
527
+ sorted_token_ids,
528
+ expert_ids,
529
+ num_tokens_post_padded,
530
+ B.shape[1],
531
+ A.shape[1],
532
+ EM,
533
+ topk_ids.numel(),
534
+ A.stride(0),
535
+ A.stride(1),
536
+ B.stride(0),
537
+ B.stride(2),
538
+ B.stride(1),
539
+ C.stride(1),
540
+ C.stride(2),
541
+ B_scale.stride(0),
542
+ B_scale.stride(2),
543
+ B_scale.stride(1),
544
+ B_zp.stride(0) if B_zp is not None else 0,
545
+ B_zp.stride(2) if B_zp is not None else 0,
546
+ B_zp.stride(1) if B_zp is not None else 0,
547
+ block_k_diviable=A.shape[1] % config["BLOCK_SIZE_K"] == 0,
548
+ group_size=block_shape[1],
549
+ MUL_ROUTED_WEIGHT=mul_routed_weight,
550
+ top_k=top_k,
551
+ compute_type=compute_type,
552
+ has_zp=B_zp is not None,
553
+ use_int4_w4a16=use_int4_w4a16,
554
+ use_int8_w8a16=use_int8_w8a16,
555
+ **config,
556
+ )
557
304
- fused_moe_kernel[grid](
305
- A,
306
- B,
307
- C,
308
- A_scale,
309
- B_scale,
310
- topk_weights,
311
- sorted_token_ids,
312
- expert_ids,
313
- num_tokens_post_padded,
314
- B.shape[1],
315
- B.shape[2],
316
- sorted_token_ids.shape[0],
317
- topk_ids.numel(),
318
- A.stride(0),
319
- A.stride(1),
320
- B.stride(0),
321
- B.stride(2),
322
- B.stride(1),
323
- C.stride(1),
324
- C.stride(2),
325
- A_scale.stride(0) if A_scale is not None and A_scale.ndim == 2 else 0,
326
- A_scale.stride(1) if A_scale is not None and A_scale.ndim == 2 else 0,
327
- B_scale.stride(0) if B_scale is not None and B_scale.ndim >= 2 else 0,
328
- B_scale.stride(2) if B_scale is not None and B_scale.ndim == 3 else 0,
329
- B_scale.stride(1) if B_scale is not None and B_scale.ndim >= 2 else 0,
330
- 0 if block_shape is None else block_shape[0],
331
- 0 if block_shape is None else block_shape[1],
332
- MUL_ROUTED_WEIGHT=mul_routed_weight,
333
- top_k=top_k,
334
- compute_type=compute_type,
335
- use_fp8_w8a8=use_fp8_w8a8,
336
- use_int8_w8a16=use_int8_w8a16,
337
- **config,
338
- )
558
+ else:
559
+ fused_moe_kernel[grid](
560
+ A,
561
+ B,
562
+ C,
563
+ A_scale,
564
+ B_scale,
565
+ topk_weights,
566
+ sorted_token_ids,
567
+ expert_ids,
568
+ num_tokens_post_padded,
569
+ B.shape[1],
570
+ A.shape[1],
571
+ EM,
572
+ topk_ids.numel(),
573
+ A.stride(0),
574
+ A.stride(1),
575
+ B.stride(0),
576
+ B.stride(2),
577
+ B.stride(1),
578
+ C.stride(1),
579
+ C.stride(2),
580
+ A_scale.stride(0)
581
+ if A_scale is not None and A_scale.ndim == 2 else 0,
582
+ A_scale.stride(1)
583
+ if A_scale is not None and A_scale.ndim == 2 else 0,
584
+ B_scale.stride(0)
585
+ if B_scale is not None and B_scale.ndim >= 2 else 0,
586
+ B_scale.stride(2)
587
+ if B_scale is not None and B_scale.ndim == 3 else 0,
588
+ B_scale.stride(1)
589
+ if B_scale is not None and B_scale.ndim >= 2 else 0,
590
+ 0 if block_shape is None else block_shape[0],
591
+ 0 if block_shape is None else block_shape[1],
592
+ MUL_ROUTED_WEIGHT=mul_routed_weight,
593
+ top_k=top_k,
594
+ compute_type=compute_type,
595
+ use_fp8_w8a8=use_fp8_w8a8,
596
+ use_int8_w8a16=use_int8_w8a16,
597
+ **config,
598
+ )
599
600
601
def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
@@ -432,7 +692,7 @@ def try_get_optimal_moe_config(
692
# NOTE: For block-wise quant,
693
# BLOCK_K must be divisible by block_shape[1]
694
# BLOCK_N and BLOCK_M has no requirements
435
- if block_shape is not None:
695
+ if block_shape is not None and block_shape[0] != 0:
696
config["BLOCK_SIZE_N"] = block_shape[0]
697
config["BLOCK_SIZE_K"] = block_shape[1]
698
return config
@@ -531,12 +791,15 @@ def grouped_topk(hidden_states: torch.Tensor,
791
792
793
def get_config_dtype_str(dtype: torch.dtype,
794
+ use_int4_w4a16: Optional[bool] = False,
795
use_int8_w8a16: Optional[bool] = False,
796
use_fp8_w8a8: Optional[bool] = False):
797
if use_fp8_w8a8:
798
return "fp8_w8a8"
799
elif use_int8_w8a16:
800
return "int8_w8a16"
801
+ elif use_int4_w4a16:
802
+ return "int4_w8a16"
803
elif dtype == torch.float:
804
# avoiding cases where kernel fails when float32 MoE
805
# use fp16/bfloat16 configs
@@ -551,14 +814,17 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
814
topk_ids: torch.Tensor,
815
use_fp8_w8a8: bool = False,
816
use_int8_w8a16: bool = False,
817
+ use_int4_w4a16: bool = False,
818
w1_scale: Optional[torch.Tensor] = None,
819
w2_scale: Optional[torch.Tensor] = None,
820
+ w1_zp: Optional[torch.Tensor] = None,
821
+ w2_zp: Optional[torch.Tensor] = None,
822
a1_scale: Optional[torch.Tensor] = None,
823
a2_scale: Optional[torch.Tensor] = None,
824
block_shape: Optional[List[int]] = None) -> None:
825
fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
560
- use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
561
- a1_scale, a2_scale, block_shape)
826
+ use_fp8_w8a8, use_int8_w8a16, use_int4_w4a16, w1_scale,
827
+ w2_scale, w1_zp, w2_zp, a1_scale, a2_scale, block_shape)
828
829
830
def inplace_fused_experts_fake(
@@ -569,8 +835,11 @@ def inplace_fused_experts_fake(
835
topk_ids: torch.Tensor,
836
use_fp8_w8a8: bool = False,
837
use_int8_w8a16: bool = False,
838
+ use_int4_w4a16: bool = False,
839
w1_scale: Optional[torch.Tensor] = None,
840
w2_scale: Optional[torch.Tensor] = None,
841
+ w1_zp: Optional[torch.Tensor] = None,
842
+ w2_zp: Optional[torch.Tensor] = None,
843
a1_scale: Optional[torch.Tensor] = None,
844
a2_scale: Optional[torch.Tensor] = None,
845
block_shape: Optional[List[int]] = None) -> None:
@@ -593,14 +862,18 @@ def outplace_fused_experts(
862
topk_ids: torch.Tensor,
863
use_fp8_w8a8: bool = False,
864
use_int8_w8a16: bool = False,
865
+ use_int4_w4a16: bool = False,
866
w1_scale: Optional[torch.Tensor] = None,
867
w2_scale: Optional[torch.Tensor] = None,
868
+ w1_zp: Optional[torch.Tensor] = None,
869
+ w2_zp: Optional[torch.Tensor] = None,
870
a1_scale: Optional[torch.Tensor] = None,
871
a2_scale: Optional[torch.Tensor] = None,
872
block_shape: Optional[List[int]] = None) -> torch.Tensor:
873
return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
602
- False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
603
- w2_scale, a1_scale, a2_scale, block_shape)
874
+ False, use_fp8_w8a8, use_int8_w8a16,
875
+ use_int4_w4a16, w1_scale, w2_scale, w1_zp, w2_zp,
876
+ a1_scale, a2_scale, block_shape)
877
878
879
def outplace_fused_experts_fake(
@@ -611,8 +884,11 @@ def outplace_fused_experts_fake(
884
topk_ids: torch.Tensor,
885
use_fp8_w8a8: bool = False,
886
use_int8_w8a16: bool = False,
887
+ use_int4_w4a16: bool = False,
888
w1_scale: Optional[torch.Tensor] = None,
889
w2_scale: Optional[torch.Tensor] = None,
890
+ w1_zp: Optional[torch.Tensor] = None,
891
+ w2_zp: Optional[torch.Tensor] = None,
892
a1_scale: Optional[torch.Tensor] = None,
893
a2_scale: Optional[torch.Tensor] = None,
894
block_shape: Optional[List[int]] = None) -> torch.Tensor:
@@ -635,8 +911,11 @@ def fused_experts(hidden_states: torch.Tensor,
911
inplace: bool = False,
912
use_fp8_w8a8: bool = False,
913
use_int8_w8a16: bool = False,
914
+ use_int4_w4a16: bool = False,
915
w1_scale: Optional[torch.Tensor] = None,
916
w2_scale: Optional[torch.Tensor] = None,
917
+ w1_zp: Optional[torch.Tensor] = None,
918
+ w2_zp: Optional[torch.Tensor] = None,
919
a1_scale: Optional[torch.Tensor] = None,
920
a2_scale: Optional[torch.Tensor] = None,
921
block_shape: Optional[List[int]] = None):
@@ -644,16 +923,15 @@ def fused_experts(hidden_states: torch.Tensor,
923
torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
924
topk_weights, topk_ids,
925
use_fp8_w8a8, use_int8_w8a16,
647
- w1_scale, w2_scale, a1_scale,
926
+ use_int4_w4a16, w1_scale,
927
+ w2_scale, w1_zp, w2_zp, a1_scale,
928
a2_scale, block_shape)
929
return hidden_states
930
else:
651
- return torch.ops.vllm.outplace_fused_experts(hidden_states, w1, w2,
652
- topk_weights, topk_ids,
653
- use_fp8_w8a8,
654
- use_int8_w8a16, w1_scale,
655
- w2_scale, a1_scale,
656
- a2_scale, block_shape)
931
+ return torch.ops.vllm.outplace_fused_experts(
932
+ hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
933
+ use_int8_w8a16, use_int4_w4a16, w1_scale, w2_scale, w1_zp, w2_zp,
934
+ a1_scale, a2_scale, block_shape)
935
936
937
def fused_experts_impl(hidden_states: torch.Tensor,
@@ -664,13 +942,21 @@ def fused_experts_impl(hidden_states: torch.Tensor,
942
inplace: bool = False,
943
use_fp8_w8a8: bool = False,
944
use_int8_w8a16: bool = False,
945
+ use_int4_w4a16: bool = False,
946
w1_scale: Optional[torch.Tensor] = None,
947
w2_scale: Optional[torch.Tensor] = None,
948
+ w1_zp: Optional[torch.Tensor] = None,
949
+ w2_zp: Optional[torch.Tensor] = None,
950
a1_scale: Optional[torch.Tensor] = None,
951
a2_scale: Optional[torch.Tensor] = None,
952
block_shape: Optional[List[int]] = None):
953
# Check constraints.
673
- assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
954
+ if use_int4_w4a16:
955
+ assert hidden_states.shape[1] // 2 == w1.shape[
956
+ 2], "Hidden size mismatch"
957
+ else:
958
+ assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
959
+
960
assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
961
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
962
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
@@ -687,6 +973,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
973
M = min(num_tokens, CHUNK_SIZE)
974
config_dtype = get_config_dtype_str(use_fp8_w8a8=use_fp8_w8a8,
975
use_int8_w8a16=use_int8_w8a16,
976
+ use_int4_w4a16=use_int4_w4a16,
977
dtype=hidden_states.dtype)
978
979
get_config_func = functools.partial(
@@ -755,6 +1042,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1042
intermediate_cache1,
1043
a1_scale,
1044
w1_scale,
1045
+ w1_zp,
1046
curr_topk_weights,
1047
curr_topk_ids,
1048
sorted_token_ids,
@@ -766,6 +1054,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1054
compute_type=compute_type,
1055
use_fp8_w8a8=use_fp8_w8a8,
1056
use_int8_w8a16=use_int8_w8a16,
1057
+ use_int4_w4a16=use_int4_w4a16,
1058
block_shape=block_shape)
1059
1060
torch.ops._C.silu_and_mul(intermediate_cache2,
@@ -776,6 +1065,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1065
intermediate_cache3,
1066
a2_scale,
1067
w2_scale,
1068
+ w2_zp,
1069
curr_topk_weights,
1070
curr_topk_ids,
1071
sorted_token_ids,
@@ -787,6 +1077,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1077
compute_type=compute_type,
1078
use_fp8_w8a8=use_fp8_w8a8,
1079
use_int8_w8a16=use_int8_w8a16,
1080
+ use_int4_w4a16=use_int4_w4a16,
1081
block_shape=block_shape)
1082
1083
ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
@@ -808,8 +1099,11 @@ def fused_moe(
1099
custom_routing_function: Optional[Callable] = None,
1100
use_fp8_w8a8: bool = False,
1101
use_int8_w8a16: bool = False,
1102
+ use_int4_w4a16: bool = False,
1103
w1_scale: Optional[torch.Tensor] = None,
1104
w2_scale: Optional[torch.Tensor] = None,
1105
+ w1_zp: Optional[torch.Tensor] = None,
1106
+ w2_zp: Optional[torch.Tensor] = None,
1107
a1_scale: Optional[torch.Tensor] = None,
1108
a2_scale: Optional[torch.Tensor] = None,
1109
block_shape: Optional[List[int]] = None,
@@ -834,8 +1128,12 @@ def fused_moe(
1128
note: Deepseekv2 model uses grouped_topk
1129
- use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner
1130
products for w1 and w2. Defaults to False.
837
- - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner
838
- products for w1 and w2. Defaults to False.
1131
+ - use_int8_w8a16 (bool): If True, use matmul of int8 weight and bf16/fp16
1132
+ activation to compute the inner products for w1 and w2.
1133
+ Defaults to False.
1134
+ - use_int4_w4a16 (bool): If True, use matmul of int4 weight and bf16/fp16
1135
+ activation to compute the inner products for w1 and w2.
1136
+ Defaults to False.
1137
- w1_scale (Optional[torch.Tensor]): Optional scale to be used for
1138
w1.
1139
- w2_scale (Optional[torch.Tensor]): Optional scale to be used for
@@ -873,8 +1171,11 @@ def fused_moe(
1171
inplace=inplace,
1172
use_fp8_w8a8=use_fp8_w8a8,
1173
use_int8_w8a16=use_int8_w8a16,
1174
+ use_int4_w4a16=use_int4_w4a16,
1175
w1_scale=w1_scale,
1176
w2_scale=w2_scale,
1177
+ w1_zp=w1_zp,
1178
+ w2_zp=w2_zp,
1179
a1_scale=a1_scale,
1180
a2_scale=a2_scale,
1181
block_shape=block_shape)
vllm/model_executor/layers/quantization/__init__.py
+5
-2
@@ -26,7 +26,8 @@ QUANTIZATION_METHODS: List[str] = [
26
"experts_int8",
27
"neuron_quant",
28
"ipex",
29
- "quark"
29
+ "quark",
30
+ "moe_wna16"
31
]
32
33
# The customized quantization methods which will be added to this dict.
@@ -94,6 +95,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
95
from .ipex_quant import IPEXConfig
96
from .marlin import MarlinConfig
97
from .modelopt import ModelOptFp8Config
98
+ from .moe_wna16 import MoeWNA16Config
99
from .neuron_quant import NeuronQuantConfig
100
from .qqq import QQQConfig
101
from .tpu_int8 import Int8TpuConfig
@@ -121,7 +123,8 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
123
"experts_int8": ExpertsInt8Config,
124
"neuron_quant": NeuronQuantConfig,
125
"ipex": IPEXConfig,
124
- "quark": QuarkConfig
126
+ "quark": QuarkConfig,
127
+ "moe_wna16": MoeWNA16Config,
128
}
129
# Update the `method_to_config` with customized quantization methods.
130
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
vllm/model_executor/layers/quantization/moe_wna16.py
new
+424
@@ -0,0 +1,424 @@
1
+from typing import Any, Callable, Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
6
+from vllm.model_executor.layers.fused_moe.layer import (
7
+ FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
8
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
9
+from vllm.model_executor.layers.quantization.awq import (AWQConfig,
10
+ AWQLinearMethod)
11
+from vllm.model_executor.layers.quantization.awq_marlin import (
12
+ AWQMarlinConfig, AWQMarlinLinearMethod)
13
+from vllm.model_executor.layers.quantization.base_config import (
14
+ QuantizationConfig, QuantizeMethodBase)
15
+from vllm.model_executor.layers.quantization.gptq import (GPTQConfig,
16
+ GPTQLinearMethod)
17
+from vllm.model_executor.layers.quantization.gptq_marlin import (
18
+ GPTQMarlinConfig, GPTQMarlinLinearMethod)
19
+from vllm.model_executor.utils import set_weight_attrs
20
+from vllm.platforms import current_platform
21
+
22
+
23
+class MoeWNA16Config(QuantizationConfig):
24
+ """Config class for MOE WNA16 (W8A16/W4A16) quantization."""
25
+
26
+ def __init__(self, linear_quant_method: str, weight_bits: int,
27
+ group_size: int, has_zp: bool, lm_head_quantized: bool,
28
+ modules_to_not_convert: Optional[List[str]],
29
+ full_config: Dict[str, Any]) -> None:
30
+ self.weight_bits = weight_bits
31
+ self.group_size = group_size
32
+ self.has_zp = has_zp
33
+ self.bit8_pack_factor = 8 // self.weight_bits
34
+ self.lm_head_quantized = lm_head_quantized
35
+ self.linear_quant_method = linear_quant_method
36
+ self.full_config = full_config
37
+ self.use_marlin = False
38
+ if self.linear_quant_method == "gptq":
39
+ self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(
40
+ full_config)
41
+ elif self.linear_quant_method == "awq":
42
+ capability_tuple = current_platform.get_device_capability()
43
+ device_capability = (-1 if capability_tuple is None else
44
+ capability_tuple.to_int())
45
+ awq_min_capability = AWQConfig.get_min_capability()
46
+ if device_capability < awq_min_capability:
47
+ raise ValueError(
48
+ "The quantization method moe_wna16 + awq is not supported "
49
+ "for the current GPU. "
50
+ f"Minimum capability: {awq_min_capability}. "
51
+ f"Current capability: {device_capability}.")
52
+ self.use_marlin = AWQMarlinConfig.is_awq_marlin_compatible(
53
+ full_config)
54
+ else:
55
+ raise ValueError("moe_wna16 only support gptq and awq.")
56
+
57
+ if modules_to_not_convert is None:
58
+ self.modules_to_not_convert = []
59
+ else:
60
+ self.modules_to_not_convert = modules_to_not_convert
61
+
62
+ @classmethod
63
+ def get_name(cls) -> str:
64
+ return "moe_wna16"
65
+
66
+ @classmethod
67
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
68
+ return [torch.bfloat16, torch.half]
69
+
70
+ @classmethod
71
+ def get_min_capability(cls) -> int:
72
+ return 70
73
+
74
+ @classmethod
75
+ def get_config_filenames(cls) -> List[str]:
76
+ return ["quantize_config.json"]
77
+
78
+ @classmethod
79
+ def from_config(cls, config: Dict[str, Any]) -> "MoeWNA16Config":
80
+ linear_quant_method = cls.get_from_keys(config, ["quant_method"])
81
+ weight_bits = cls.get_from_keys(config, ["bits"])
82
+ group_size = cls.get_from_keys(config, ["group_size"])
83
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
84
+ default=False)
85
+ if linear_quant_method == "gptq":
86
+ has_zp = not cls.get_from_keys(config, ["sym"])
87
+ modules_to_not_convert = []
88
+ elif linear_quant_method == "awq":
89
+ has_zp = cls.get_from_keys(config, ["zero_point"])
90
+ modules_to_not_convert = cls.get_from_keys(
91
+ config, ["modules_to_not_convert"])
92
+ else:
93
+ raise ValueError("moe_wna16 only support gptq and awq.")
94
+
95
+ return cls(linear_quant_method, weight_bits, group_size, has_zp,
96
+ lm_head_quantized, modules_to_not_convert, config)
97
+
98
+ @classmethod
99
+ def override_quantization_method(cls, hf_quant_cfg,
100
+ user_quant) -> Optional[str]:
101
+ can_convert = cls.is_moe_wna16_compatible(hf_quant_cfg)
102
+ if can_convert and user_quant == "moe_wna16":
103
+ return cls.get_name()
104
+ return None
105
+
106
+ @classmethod
107
+ def is_moe_wna16_compatible(cls, quant_config: Dict[str, Any]):
108
+ # Extract data from quant config.
109
+ quant_method = quant_config.get("quant_method", "").lower()
110
+ num_bits = quant_config.get("bits")
111
+ desc_act = quant_config.get("desc_act")
112
+
113
+ capability_tuple = current_platform.get_device_capability()
114
+ device_capability = (-1 if capability_tuple is None else
115
+ capability_tuple.to_int())
116
+ awq_min_capability = AWQConfig.get_min_capability()
117
+
118
+ gptq_compatible = quant_method == "gptq" and \
119
+ not desc_act and num_bits in [4, 8]
120
+ awq_compatible = quant_method == "awq" and num_bits == 4 and \
121
+ device_capability >= awq_min_capability
122
+
123
+ return gptq_compatible or awq_compatible
124
+
125
+ def get_quant_method(self, layer: torch.nn.Module,
126
+ prefix: str) -> Optional["QuantizeMethodBase"]:
127
+ if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
128
+ return UnquantizedLinearMethod()
129
+ elif isinstance(layer, FusedMoE):
130
+ return MoeWNA16Method(self)
131
+ else:
132
+ if self.linear_quant_method == "gptq":
133
+ if self.use_marlin:
134
+ return GPTQMarlinLinearMethod(
135
+ GPTQMarlinConfig.from_config(self.full_config))
136
+ else:
137
+ return GPTQLinearMethod(
138
+ GPTQConfig.from_config(self.full_config))
139
+ elif self.linear_quant_method == "awq":
140
+ if self.use_marlin:
141
+ return AWQMarlinLinearMethod(
142
+ AWQMarlinConfig.from_config(self.full_config))
143
+ else:
144
+ return AWQLinearMethod(
145
+ AWQConfig.from_config(self.full_config))
146
+ else:
147
+ raise ValueError("moe_wna16 only support gptq and awq.")
148
+
149
+
150
+def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]):
151
+ return any(module_name in prefix for module_name in modules_to_not_convert)
152
+
153
+
154
+class MoeWNA16Method(FusedMoEMethodBase):
155
+ """Linear method for MOE WNA16 (W8A16/W4A16) quantization.
156
+
157
+ Args:
158
+ quant_config: The MOE WNA16 (W8A16/W4A16) quantization config.
159
+ """
160
+
161
+ def __init__(self, quant_config: MoeWNA16Config):
162
+ self.quant_config = quant_config
163
+
164
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
165
+ hidden_size: int, intermediate_size_per_partition: int,
166
+ params_dtype: torch.dtype, **extra_weight_attrs):
167
+
168
+ layer.quant_config = self.quant_config
169
+ bit8_pack_factor = self.quant_config.bit8_pack_factor
170
+ group_size = self.quant_config.group_size
171
+ group_size_div_factor = 1
172
+
173
+ # make intermediate_size and hidden_size diviable by group_size
174
+ # we reduce the group size to ensure that
175
+ # and we would repeat the loaded_weight later
176
+ while intermediate_size_per_partition % group_size or \
177
+ hidden_size % group_size:
178
+ group_size = group_size // 2
179
+ group_size_div_factor *= 2
180
+ assert group_size >= 32
181
+ layer.group_size = group_size
182
+ layer.group_size_div_factor = group_size_div_factor
183
+
184
+ strategy = FusedMoeWeightScaleSupported.GROUP.value
185
+ extra_weight_attrs.update({
186
+ "quant_method": strategy,
187
+ "is_transposed": False
188
+ })
189
+
190
+ assert 'weight_loader' in extra_weight_attrs
191
+ weight_loader = extra_weight_attrs['weight_loader']
192
+ wrapped_weight_loader = MoeWNA16Method.get_weight_loader(
193
+ layer, weight_loader)
194
+ extra_weight_attrs['weight_loader'] = wrapped_weight_loader
195
+
196
+ # Fused gate_up_proj (column parallel)
197
+ w13_qweight = torch.nn.Parameter(torch.empty(
198
+ num_experts,
199
+ 2 * intermediate_size_per_partition,
200
+ hidden_size // bit8_pack_factor,
201
+ dtype=torch.uint8),
202
+ requires_grad=False)
203
+ layer.register_parameter("w13_qweight", w13_qweight)
204
+ set_weight_attrs(w13_qweight, extra_weight_attrs)
205
+
206
+ # down_proj (row parallel)
207
+ w2_qweight = torch.nn.Parameter(torch.empty(
208
+ num_experts,
209
+ hidden_size,
210
+ intermediate_size_per_partition // bit8_pack_factor,
211
+ dtype=torch.uint8),
212
+ requires_grad=False)
213
+ layer.register_parameter("w2_qweight", w2_qweight)
214
+ set_weight_attrs(w2_qweight, extra_weight_attrs)
215
+
216
+ w13_scales = torch.nn.Parameter(torch.zeros(
217
+ num_experts,
218
+ 2 * intermediate_size_per_partition,
219
+ hidden_size // group_size,
220
+ dtype=params_dtype),
221
+ requires_grad=False)
222
+ layer.register_parameter("w13_scales", w13_scales)
223
+ set_weight_attrs(w13_scales, extra_weight_attrs)
224
+
225
+ w2_scales = torch.nn.Parameter(torch.zeros(
226
+ num_experts,
227
+ hidden_size,
228
+ intermediate_size_per_partition // group_size,
229
+ dtype=params_dtype),
230
+ requires_grad=False)
231
+ layer.register_parameter("w2_scales", w2_scales)
232
+ set_weight_attrs(w2_scales, extra_weight_attrs)
233
+
234
+ if self.quant_config.has_zp:
235
+ w13_qzeros = torch.nn.Parameter(torch.zeros(
236
+ num_experts,
237
+ 2 * intermediate_size_per_partition // bit8_pack_factor,
238
+ hidden_size // group_size,
239
+ dtype=torch.uint8),
240
+ requires_grad=False)
241
+ layer.register_parameter("w13_qzeros", w13_qzeros)
242
+ set_weight_attrs(w13_qzeros, extra_weight_attrs)
243
+
244
+ w2_qzeros = torch.nn.Parameter(torch.zeros(
245
+ num_experts,
246
+ hidden_size // bit8_pack_factor,
247
+ intermediate_size_per_partition // group_size,
248
+ dtype=torch.uint8),
249
+ requires_grad=False)
250
+ layer.register_parameter("w2_qzeros", w2_qzeros)
251
+ set_weight_attrs(w2_qzeros, extra_weight_attrs)
252
+
253
+ if self.quant_config.linear_quant_method == "gptq":
254
+ # some param are unused, but we need to init them in order to
255
+ # load weights
256
+ invalid_param_keys = ["w13_g_idx", "w2_g_idx"]
257
+ if not self.quant_config.has_zp:
258
+ invalid_param_keys += ["w13_qzeros", "w2_qzeros"]
259
+ for key in invalid_param_keys:
260
+ param = torch.nn.Parameter(torch.empty((0, ),
261
+ dtype=torch.int32),
262
+ requires_grad=False)
263
+ layer.register_parameter(key, param)
264
+ set_weight_attrs(param, extra_weight_attrs)
265
+
266
+ def apply(
267
+ self,
268
+ layer: torch.nn.Module,
269
+ x: torch.Tensor,
270
+ router_logits: torch.Tensor,
271
+ top_k: int,
272
+ renormalize: bool,
273
+ use_grouped_topk: bool = False,
274
+ topk_group: Optional[int] = None,
275
+ num_expert_group: Optional[int] = None,
276
+ custom_routing_function: Optional[Callable] = None,
277
+ scoring_func: str = "softmax",
278
+ e_score_correction_bias: Optional[torch.Tensor] = None,
279
+ ) -> torch.Tensor:
280
+ from vllm.model_executor.layers.fused_moe import fused_experts
281
+
282
+ topk_weights, topk_ids = FusedMoE.select_experts(
283
+ hidden_states=x,
284
+ router_logits=router_logits,
285
+ use_grouped_topk=use_grouped_topk,
286
+ top_k=top_k,
287
+ renormalize=renormalize,
288
+ topk_group=topk_group,
289
+ num_expert_group=num_expert_group,
290
+ custom_routing_function=custom_routing_function,
291
+ scoring_func=scoring_func,
292
+ e_score_correction_bias=e_score_correction_bias)
293
+
294
+ weight_bits = self.quant_config.weight_bits
295
+ has_zp = self.quant_config.has_zp
296
+
297
+ return fused_experts(x,
298
+ layer.w13_qweight,
299
+ layer.w2_qweight,
300
+ topk_weights=topk_weights,
301
+ topk_ids=topk_ids,
302
+ inplace=True,
303
+ use_int4_w4a16=weight_bits == 4,
304
+ use_int8_w8a16=weight_bits == 8,
305
+ w1_scale=layer.w13_scales,
306
+ w2_scale=layer.w2_scales,
307
+ w1_zp=layer.w13_qzeros if has_zp else None,
308
+ w2_zp=layer.w2_qzeros if has_zp else None,
309
+ block_shape=[0, layer.group_size])
310
+
311
+ @staticmethod
312
+ def get_weight_loader(layer, weight_loader):
313
+
314
+ def convert_awq_tensor(tensor, tensor_type):
315
+ # convert awq qweight/qzeros to a standard format (assume int4)
316
+ # qweight: (k, n // pack_factor_bit32) -> (n, k // pack_factor_bit8)
317
+ # qzeros: (k // group_size, n // pack_factor_bit32) ->
318
+ # (n // pack_factor_bit8, k // group_size)
319
+ # pack_factor_bit32 = 32 // weight_bits
320
+ # pack_factor_bit8 = 8 // weight_bits
321
+
322
+ # 0. suppose origin shape (a, b), dtype int32
323
+ # 1. convert to uint8, shape (a, b) -> (a, 4 * b)
324
+ size0 = tensor.size(0)
325
+ tensor = tensor.view(torch.uint8)
326
+
327
+ # 2. unpack to uint4 (only when weight_bits == 4)
328
+ # shape (a, 4 * b) -> (a, 4 * b, 2)
329
+ shifter = torch.tensor([0, 4],
330
+ dtype=torch.uint8,
331
+ device=tensor.device)
332
+ tensor = (tensor[:, :, None] >> shifter) & 0xF
333
+
334
+ # 3. change order, see
335
+ # https://github.com/casper-hansen/AutoAWQ/blob/v0.2.8/awq/utils/quant_utils.py
336
+ # shape -> (a, 4 * b * pack_factor_bit8)
337
+ reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
338
+ tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
339
+ tensor = tensor.view(size0, -1)
340
+
341
+ # 4. transpose, shape -> (4 * b * pack_factor_bit8, a)
342
+ tensor = tensor.T.contiguous()
343
+
344
+ # 5. repack (only when weight_bits == 4)
345
+ # qweight shape -> (4 * b * pack_factor_bit8, a // pack_factor_bit8)
346
+ # qzeros shape -> (4 * b, a)
347
+
348
+ if tensor_type == "qweight":
349
+ tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
350
+ elif tensor_type == "qzeros":
351
+ tensor = tensor[1::2, :] * 16 + tensor[::2, :]
352
+ return tensor
353
+
354
+ def convert_gptq_int4_qzeros(tensor):
355
+ tensor = tensor.view(torch.uint8)
356
+ shifter = torch.tensor([0, 4],
357
+ dtype=torch.uint8,
358
+ device=tensor.device)
359
+ tensor = (tensor[:, :, None] >> shifter) & 0xF
360
+ tensor = tensor + 1
361
+ tensor = tensor[:, :, 0] + tensor[:, :, 1] * 16
362
+ return tensor
363
+
364
+ def moe_wna16_weight_loader(param: torch.nn.Parameter,
365
+ loaded_weight: torch.Tensor,
366
+ weight_name: str, shard_id: str,
367
+ expert_id: int):
368
+ if "g_idx" in weight_name:
369
+ return
370
+ if not layer.quant_config.has_zp and "qzeros" in weight_name:
371
+ return
372
+
373
+ device = get_tp_group().device
374
+ tp_rank = get_tensor_model_parallel_rank()
375
+ loaded_weight = loaded_weight.to(device)
376
+ shard_size = layer.intermediate_size_per_partition
377
+
378
+ # convert gptq and awq weight to a standard format
379
+ if layer.quant_config.linear_quant_method == "awq":
380
+ assert layer.quant_config.weight_bits == 4
381
+ if "weight" in weight_name:
382
+ loaded_weight = convert_awq_tensor(loaded_weight,
383
+ "qweight")
384
+ elif "zeros" in weight_name:
385
+ loaded_weight = convert_awq_tensor(loaded_weight, "qzeros")
386
+ else:
387
+ loaded_weight = loaded_weight.T
388
+ elif layer.quant_config.linear_quant_method == "gptq":
389
+ assert layer.quant_config.weight_bits in [4, 8]
390
+ if "weight" in weight_name:
391
+ loaded_weight = loaded_weight.T.contiguous().view(
392
+ torch.uint8)
393
+ elif "zeros" in weight_name:
394
+ # add 1 to gptq qzeros to align with awq
395
+ loaded_weight = loaded_weight.view(torch.uint8)
396
+ if layer.quant_config.weight_bits == 4:
397
+ loaded_weight = convert_gptq_int4_qzeros(
398
+ loaded_weight).T
399
+ else:
400
+ loaded_weight = loaded_weight.T + 1
401
+ else:
402
+ loaded_weight = loaded_weight.T
403
+
404
+ # repeat the qzeros/scales to fit new group size
405
+ if layer.group_size_div_factor > 1 and \
406
+ "qzeros" in weight_name or "scales" in weight_name:
407
+ loaded_weight = loaded_weight.repeat_interleave(
408
+ layer.group_size_div_factor, 1)
409
+
410
+ if "w13_qzeros" in weight_name:
411
+ tensor = loaded_weight.view(layer.tp_size, -1,
412
+ loaded_weight.size(1))[tp_rank]
413
+ if shard_id == "w1":
414
+ param.data[expert_id, :shard_size // 2] = tensor
415
+ else:
416
+ param.data[expert_id, shard_size // 2:] = tensor
417
+ elif "w2_qzeros" in weight_name:
418
+ param.data[expert_id] = loaded_weight.view(
419
+ loaded_weight.size(0), layer.tp_size, -1)[:, tp_rank]
420
+ else:
421
+ weight_loader(param, loaded_weight, weight_name, shard_id,
422
+ expert_id)
423
+
424
+ return moe_wna16_weight_loader