@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27b78c73ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Kernel] add triton fused moe kernel for gptq/awq (#12185)

Jinzhen Lin committed Jan 29, 2025 at 22:07 UTC 27b78c73cad00f5c7bb3b2431f02dc680f7034bc
4 files changed +874 -55
tests/kernels/test_moe.py
+91
@@ -18,6 +18,8 @@ from vllm.model_executor.layers.fused_moe.moe_torch_iterative import (
18 fused_moe as iterative_moe)
19 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20 marlin_quantize)
21 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
22 + quantize_weights)
23 from vllm.model_executor.models.mixtral import MixtralMoE
24 from vllm.platforms import current_platform
25 from vllm.scalar_type import scalar_types
@@ -55,6 +57,95 @@ def test_fused_moe(
57 rtol=0)
58
59
60 +@pytest.mark.parametrize("m", [1, 32, 222])
61 +@pytest.mark.parametrize("n", [128, 1024, 2048])
62 +@pytest.mark.parametrize("k", [128, 1024])
63 +@pytest.mark.parametrize("e", NUM_EXPERTS)
64 +@pytest.mark.parametrize("topk", TOP_KS)
65 +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
66 +@pytest.mark.parametrize("group_size", [64, 128])
67 +@pytest.mark.parametrize("has_zp", [True, False])
68 +@pytest.mark.parametrize("weight_bits", [4, 8])
69 +def test_fused_moe_wn16(m: int, n: int, k: int, e: int, topk: int,
70 + dtype: torch.dtype, group_size: int, has_zp: bool,
71 + weight_bits: int):
72 + print(m, n, k, e, topk, dtype, group_size, has_zp, weight_bits)
73 + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
74 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
75 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
76 + score = torch.randn((m, e), device="cuda", dtype=dtype)
77 +
78 + if weight_bits == 4:
79 + pack_factor = 2
80 + quant_type = scalar_types.uint4 if has_zp else scalar_types.uint4b8
81 + elif weight_bits == 8:
82 + pack_factor = 1
83 + quant_type = scalar_types.uint8 if has_zp else scalar_types.uint8b128
84 +
85 + w1_ref = w1.clone()
86 + w2_ref = w2.clone()
87 + w1_qweight = torch.empty((e, 2 * n, k // pack_factor),
88 + device="cuda",
89 + dtype=torch.uint8)
90 + w2_qweight = torch.empty((e, k, n // pack_factor),
91 + device="cuda",
92 + dtype=torch.uint8)
93 + w1_scales = torch.empty((e, 2 * n, k // group_size),
94 + device="cuda",
95 + dtype=dtype)
96 + w2_scales = torch.empty((e, k, n // group_size),
97 + device="cuda",
98 + dtype=dtype)
99 + w1_qzeros = torch.empty((e, 2 * n // pack_factor, k // group_size),
100 + device="cuda",
101 + dtype=torch.uint8)
102 + w2_qzeros = torch.empty((e, k // pack_factor, n // group_size),
103 + device="cuda",
104 + dtype=torch.uint8)
105 +
106 + for i in range(e * 2):
107 + expert_id = i % e
108 + if i // e == 0:
109 + w, w_ref, w_qweight, w_scales, w_qzeros = \
110 + w1, w1_ref, w1_qweight, w1_scales, w1_qzeros
111 + else:
112 + w, w_ref, w_qweight, w_scales, w_qzeros = \
113 + w2, w2_ref, w2_qweight, w2_scales, w2_qzeros
114 + weight, qweight, scales, qzeros = quantize_weights(
115 + w[expert_id].T, quant_type, group_size, has_zp, False)
116 + weight = weight.T
117 + qweight = qweight.T.contiguous().to(torch.uint8)
118 + scales = scales.T
119 + if has_zp:
120 + qzeros = qzeros.T.contiguous().to(torch.uint8)
121 + if weight_bits == 4:
122 + qweight = qweight[:, 1::2] * 16 + qweight[:, ::2]
123 + if has_zp:
124 + qzeros = qzeros[1::2, :] * 16 + qzeros[::2, :]
125 +
126 + w_ref[expert_id] = weight
127 + w_qweight[expert_id] = qweight
128 + w_scales[expert_id] = scales
129 + if has_zp:
130 + w_qzeros[expert_id] = qzeros
131 +
132 + triton_output = fused_moe(a,
133 + w1_qweight,
134 + w2_qweight,
135 + score,
136 + topk,
137 + renormalize=False,
138 + use_int4_w4a16=weight_bits == 4,
139 + use_int8_w8a16=weight_bits == 8,
140 + w1_scale=w1_scales,
141 + w2_scale=w2_scales,
142 + w1_zp=w1_qzeros if has_zp else None,
143 + w2_zp=w2_qzeros if has_zp else None,
144 + block_shape=[0, group_size])
145 + torch_output = torch_moe(a, w1_ref, w2_ref, score, topk)
146 + torch.testing.assert_close(triton_output, torch_output, atol=2e-2, rtol=0)
147 +
148 +
149 @pytest.mark.parametrize("dtype",
150 [torch.float32, torch.float16, torch.bfloat16])
151 @torch.inference_mode()
vllm/model_executor/layers/fused_moe/fused_moe.py
+354 -53
@@ -19,6 +19,206 @@ from vllm.utils import direct_register_custom_op
19 logger = init_logger(__name__)
20
21
22 +@triton.jit
23 +def fused_moe_kernel_gptq_awq(
24 + # Pointers to matrices
25 + a_ptr,
26 + b_ptr,
27 + c_ptr,
28 + b_scale_ptr,
29 + b_zp_ptr,
30 + topk_weights_ptr,
31 + sorted_token_ids_ptr,
32 + expert_ids_ptr,
33 + num_tokens_post_padded_ptr,
34 + # Matrix dimensions
35 + N: tl.constexpr,
36 + K: tl.constexpr,
37 + EM,
38 + num_valid_tokens,
39 + # The stride variables represent how much to increase the ptr by when
40 + # moving by 1 element in a particular dimension. E.g. `stride_am` is
41 + # how much to increase `a_ptr` by to get the element one row down
42 + # (A has M rows).
43 + stride_am,
44 + stride_ak,
45 + stride_be,
46 + stride_bk,
47 + stride_bn,
48 + stride_cm,
49 + stride_cn,
50 + stride_bse,
51 + stride_bsk,
52 + stride_bsn,
53 + stride_bze,
54 + stride_bzk,
55 + stride_bzn,
56 + block_k_diviable: tl.constexpr,
57 + group_size: tl.constexpr,
58 + # Meta-parameters
59 + BLOCK_SIZE_M: tl.constexpr,
60 + BLOCK_SIZE_N: tl.constexpr,
61 + BLOCK_SIZE_K: tl.constexpr,
62 + GROUP_SIZE_M: tl.constexpr,
63 + MUL_ROUTED_WEIGHT: tl.constexpr,
64 + top_k: tl.constexpr,
65 + compute_type: tl.constexpr,
66 + has_zp: tl.constexpr,
67 + use_int4_w4a16: tl.constexpr,
68 + use_int8_w8a16: tl.constexpr):
69 + """
70 + Implements the fused computation for a Mixture of Experts (MOE) using
71 + token and expert matrices.
72 +
73 + Key Parameters:
74 + - A: The input tensor representing tokens with shape (*, K), where '*' can
75 + be any shape representing batches and K is the feature dimension of
76 + each token.
77 + - B: The stacked MOE weight tensor with shape (E, N, K), where E is
78 + the number of experts, K is the input feature dimension, and N is
79 + the output feature dimension.
80 + - C: The output cache tensor with shape (M, topk, N), where M is the
81 + total number of tokens post padding, topk is the number of times
82 + each token is repeated, and N is the output feature dimension.
83 + - sorted_token_ids: A tensor containing the sorted indices of tokens,
84 + repeated topk times and arranged by the expert index they are
85 + assigned to.
86 + - expert_ids: A tensor containing the indices of the expert for each
87 + block. It determines which expert matrix from B should be used for
88 + each block in A.
89 + This kernel performs the multiplication of a token by its corresponding
90 + expert matrix as determined by `expert_ids`. The sorting of
91 + `sorted_token_ids` by expert index and padding ensures divisibility by
92 + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix
93 + multiplication across different blocks processed by the same expert.
94 + """
95 + # -----------------------------------------------------------
96 + # Map program ids `pid` to the block of C it should compute.
97 + # This is done in a grouped ordering to promote L2 data reuse.
98 + pid = tl.program_id(axis=0)
99 + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
100 + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
101 + num_pid_in_group = GROUP_SIZE_M * num_pid_n
102 + group_id = pid // num_pid_in_group
103 + first_pid_m = group_id * GROUP_SIZE_M
104 + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
105 + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
106 + pid_n = (pid % num_pid_in_group) // group_size_m
107 +
108 + # ----------------------------------------------------------
109 + # Create pointers for the first blocks of A and B.
110 + # We will advance this pointer as we move in the K direction
111 + # and accumulate
112 + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers
113 + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers
114 + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
115 + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
116 + return
117 + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(
118 + tl.int64)
119 + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
120 + token_mask = offs_token < num_valid_tokens
121 +
122 + offs_bn = (pid_n * BLOCK_SIZE_N +
123 + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
124 + offs_k = tl.arange(0, BLOCK_SIZE_K)
125 + a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
126 + offs_k[None, :] * stride_ak)
127 +
128 + off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
129 +
130 + if use_int4_w4a16:
131 + b_ptrs = b_ptr + off_experts * stride_be + \
132 + (offs_k[:, None] // 2) * stride_bk + offs_bn[None, :] * stride_bn
133 + b_shifter = (offs_k[:, None] % 2) * 4
134 + elif use_int8_w8a16:
135 + b_ptrs = b_ptr + off_experts * stride_be + \
136 + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn
137 +
138 + if not has_zp and use_int4_w4a16:
139 + b_zp_num = 8
140 + if not has_zp and use_int8_w8a16:
141 + b_zp_num = 128
142 + elif has_zp and use_int4_w4a16:
143 + b_zp_shifter = (offs_bn[None, :] % 2) * 4
144 +
145 + # -----------------------------------------------------------
146 + # Iterate to compute a block of the C matrix.
147 + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
148 + # of fp32 values for higher accuracy.
149 + # `accumulator` will be converted back to fp16 after the loop.
150 + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
151 + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
152 + # Load the next block of A and B, generate a mask by checking the
153 + # K dimension.
154 +
155 + if not block_k_diviable:
156 + k_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K
157 + k_other = 0.0
158 + else:
159 + k_mask = None
160 + k_other = None
161 +
162 + a = tl.load(a_ptrs,
163 + mask=token_mask[:, None] &
164 + (offs_k[None, :] < K - k * BLOCK_SIZE_K),
165 + other=0.0)
166 + b = tl.load(b_ptrs)
167 + if use_int4_w4a16:
168 + b = (b >> b_shifter) & 0xF
169 +
170 + b_scale_ptrs = b_scale_ptr + off_experts * stride_bse + \
171 + offs_bn[None, :] * stride_bsn + \
172 + ((offs_k[:, None] + BLOCK_SIZE_K * k) // group_size) * stride_bsk
173 + b_scale = tl.load(b_scale_ptrs, mask=k_mask, other=k_other)
174 + b_scale = b_scale.to(tl.float32)
175 +
176 + if has_zp and use_int4_w4a16:
177 + offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size
178 + b_zp_ptrs = b_zp_ptr + off_experts * stride_bze + \
179 + (offs_bn[None, :] // 2) * stride_bzn + \
180 + offs_k_true * stride_bzk
181 + b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other)
182 + b_zp = ((b_zp >> b_zp_shifter) & 0xF)
183 + b_zp = b_zp.to(tl.float32)
184 + elif has_zp and use_int8_w8a16:
185 + offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size
186 + b_zp_ptrs = b_zp_ptr + off_experts * stride_bze + \
187 + offs_bn[None, :] * stride_bzn + \
188 + offs_k_true * stride_bzk
189 + b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other)
190 + b_zp = b_zp.to(tl.float32)
191 +
192 + # We accumulate along the K dimension.
193 + if has_zp:
194 + b = ((b.to(tl.float32) - b_zp) * b_scale).to(compute_type)
195 + else:
196 + b = ((b.to(tl.float32) - b_zp_num) * b_scale).to(compute_type)
197 + accumulator = tl.dot(a, b, acc=accumulator)
198 +
199 + # Advance the ptrs to the next K block.
200 + a_ptrs += BLOCK_SIZE_K * stride_ak
201 + if use_int4_w4a16:
202 + b_ptrs += (BLOCK_SIZE_K // 2) * stride_bk
203 + else:
204 + b_ptrs += BLOCK_SIZE_K * stride_bk
205 +
206 + if MUL_ROUTED_WEIGHT:
207 + moe_weight = tl.load(topk_weights_ptr + offs_token,
208 + mask=token_mask,
209 + other=0)
210 + accumulator = accumulator * moe_weight[:, None]
211 +
212 + accumulator = accumulator.to(compute_type)
213 + # -----------------------------------------------------------
214 + # Write back the block of the output
215 + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
216 + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[
217 + None, :]
218 + c_mask = token_mask[:, None] & (offs_cn[None, :] < N)
219 + tl.store(c_ptrs, accumulator, mask=c_mask)
220 +
221 +
222 @triton.jit
223 def fused_moe_kernel(
224 # Pointers to matrices
@@ -266,6 +466,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
466 C: torch.Tensor,
467 A_scale: Optional[torch.Tensor],
468 B_scale: Optional[torch.Tensor],
469 + B_zp: Optional[torch.Tensor],
470 topk_weights: torch.Tensor,
471 topk_ids: torch.Tensor,
472 sorted_token_ids: torch.Tensor,
@@ -277,6 +478,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
478 compute_type: tl.dtype,
479 use_fp8_w8a8: bool,
480 use_int8_w8a16: bool,
481 + use_int4_w4a16: bool,
482 block_shape: Optional[List[int]] = None) -> None:
483 assert topk_weights.stride(1) == 1
484 assert sorted_token_ids.stride(0) == 1
@@ -292,50 +494,108 @@ def invoke_fused_moe_kernel(A: torch.Tensor,
494 assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1]
495 assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2]
496 assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1]
295 - elif use_int8_w8a16:
497 + elif use_int8_w8a16 or use_int4_w4a16:
498 assert B_scale is not None
499 + assert block_shape is None or block_shape[0] == 0
500 else:
501 assert A_scale is None
502 assert B_scale is None
503
301 - grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
302 - 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
504 + EM = sorted_token_ids.shape[0]
505 + if A.shape[0] < config["BLOCK_SIZE_M"]:
506 + # optimize for small batch_size.
507 + # We assume that top_ids of each token is unique, so
508 + # so num_valid_experts <= batch_size <= BLOCK_SIZE_M,
509 + # and we can skip some invalid blocks.
510 + EM = min(sorted_token_ids.shape[0],
511 + A.shape[0] * top_k * config['BLOCK_SIZE_M'])
512 + grid = lambda META: (triton.cdiv(EM, META['BLOCK_SIZE_M']) * triton.cdiv(
513 + B.shape[1], META['BLOCK_SIZE_N']), )
514 +
515 + if (use_int8_w8a16 or use_int4_w4a16) and \
516 + block_shape is not None and block_shape[1] > 0:
517 + assert B_scale is not None and B_scale.ndim == 3
518 + assert B_zp is None or B_zp.ndim == 3
519 +
520 + fused_moe_kernel_gptq_awq[grid](
521 + A,
522 + B,
523 + C,
524 + B_scale,
525 + B_zp,
526 + topk_weights,
527 + sorted_token_ids,
528 + expert_ids,
529 + num_tokens_post_padded,
530 + B.shape[1],
531 + A.shape[1],
532 + EM,
533 + topk_ids.numel(),
534 + A.stride(0),
535 + A.stride(1),
536 + B.stride(0),
537 + B.stride(2),
538 + B.stride(1),
539 + C.stride(1),
540 + C.stride(2),
541 + B_scale.stride(0),
542 + B_scale.stride(2),
543 + B_scale.stride(1),
544 + B_zp.stride(0) if B_zp is not None else 0,
545 + B_zp.stride(2) if B_zp is not None else 0,
546 + B_zp.stride(1) if B_zp is not None else 0,
547 + block_k_diviable=A.shape[1] % config["BLOCK_SIZE_K"] == 0,
548 + group_size=block_shape[1],
549 + MUL_ROUTED_WEIGHT=mul_routed_weight,
550 + top_k=top_k,
551 + compute_type=compute_type,
552 + has_zp=B_zp is not None,
553 + use_int4_w4a16=use_int4_w4a16,
554 + use_int8_w8a16=use_int8_w8a16,
555 + **config,
556 + )
557
304 - fused_moe_kernel[grid](
305 - A,
306 - B,
307 - C,
308 - A_scale,
309 - B_scale,
310 - topk_weights,
311 - sorted_token_ids,
312 - expert_ids,
313 - num_tokens_post_padded,
314 - B.shape[1],
315 - B.shape[2],
316 - sorted_token_ids.shape[0],
317 - topk_ids.numel(),
318 - A.stride(0),
319 - A.stride(1),
320 - B.stride(0),
321 - B.stride(2),
322 - B.stride(1),
323 - C.stride(1),
324 - C.stride(2),
325 - A_scale.stride(0) if A_scale is not None and A_scale.ndim == 2 else 0,
326 - A_scale.stride(1) if A_scale is not None and A_scale.ndim == 2 else 0,
327 - B_scale.stride(0) if B_scale is not None and B_scale.ndim >= 2 else 0,
328 - B_scale.stride(2) if B_scale is not None and B_scale.ndim == 3 else 0,
329 - B_scale.stride(1) if B_scale is not None and B_scale.ndim >= 2 else 0,
330 - 0 if block_shape is None else block_shape[0],
331 - 0 if block_shape is None else block_shape[1],
332 - MUL_ROUTED_WEIGHT=mul_routed_weight,
333 - top_k=top_k,
334 - compute_type=compute_type,
335 - use_fp8_w8a8=use_fp8_w8a8,
336 - use_int8_w8a16=use_int8_w8a16,
337 - **config,
338 - )
558 + else:
559 + fused_moe_kernel[grid](
560 + A,
561 + B,
562 + C,
563 + A_scale,
564 + B_scale,
565 + topk_weights,
566 + sorted_token_ids,
567 + expert_ids,
568 + num_tokens_post_padded,
569 + B.shape[1],
570 + A.shape[1],
571 + EM,
572 + topk_ids.numel(),
573 + A.stride(0),
574 + A.stride(1),
575 + B.stride(0),
576 + B.stride(2),
577 + B.stride(1),
578 + C.stride(1),
579 + C.stride(2),
580 + A_scale.stride(0)
581 + if A_scale is not None and A_scale.ndim == 2 else 0,
582 + A_scale.stride(1)
583 + if A_scale is not None and A_scale.ndim == 2 else 0,
584 + B_scale.stride(0)
585 + if B_scale is not None and B_scale.ndim >= 2 else 0,
586 + B_scale.stride(2)
587 + if B_scale is not None and B_scale.ndim == 3 else 0,
588 + B_scale.stride(1)
589 + if B_scale is not None and B_scale.ndim >= 2 else 0,
590 + 0 if block_shape is None else block_shape[0],
591 + 0 if block_shape is None else block_shape[1],
592 + MUL_ROUTED_WEIGHT=mul_routed_weight,
593 + top_k=top_k,
594 + compute_type=compute_type,
595 + use_fp8_w8a8=use_fp8_w8a8,
596 + use_int8_w8a16=use_int8_w8a16,
597 + **config,
598 + )
599
600
601 def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
@@ -432,7 +692,7 @@ def try_get_optimal_moe_config(
692 # NOTE: For block-wise quant,
693 # BLOCK_K must be divisible by block_shape[1]
694 # BLOCK_N and BLOCK_M has no requirements
435 - if block_shape is not None:
695 + if block_shape is not None and block_shape[0] != 0:
696 config["BLOCK_SIZE_N"] = block_shape[0]
697 config["BLOCK_SIZE_K"] = block_shape[1]
698 return config
@@ -531,12 +791,15 @@ def grouped_topk(hidden_states: torch.Tensor,
791
792
793 def get_config_dtype_str(dtype: torch.dtype,
794 + use_int4_w4a16: Optional[bool] = False,
795 use_int8_w8a16: Optional[bool] = False,
796 use_fp8_w8a8: Optional[bool] = False):
797 if use_fp8_w8a8:
798 return "fp8_w8a8"
799 elif use_int8_w8a16:
800 return "int8_w8a16"
801 + elif use_int4_w4a16:
802 + return "int4_w8a16"
803 elif dtype == torch.float:
804 # avoiding cases where kernel fails when float32 MoE
805 # use fp16/bfloat16 configs
@@ -551,14 +814,17 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
814 topk_ids: torch.Tensor,
815 use_fp8_w8a8: bool = False,
816 use_int8_w8a16: bool = False,
817 + use_int4_w4a16: bool = False,
818 w1_scale: Optional[torch.Tensor] = None,
819 w2_scale: Optional[torch.Tensor] = None,
820 + w1_zp: Optional[torch.Tensor] = None,
821 + w2_zp: Optional[torch.Tensor] = None,
822 a1_scale: Optional[torch.Tensor] = None,
823 a2_scale: Optional[torch.Tensor] = None,
824 block_shape: Optional[List[int]] = None) -> None:
825 fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
560 - use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
561 - a1_scale, a2_scale, block_shape)
826 + use_fp8_w8a8, use_int8_w8a16, use_int4_w4a16, w1_scale,
827 + w2_scale, w1_zp, w2_zp, a1_scale, a2_scale, block_shape)
828
829
830 def inplace_fused_experts_fake(
@@ -569,8 +835,11 @@ def inplace_fused_experts_fake(
835 topk_ids: torch.Tensor,
836 use_fp8_w8a8: bool = False,
837 use_int8_w8a16: bool = False,
838 + use_int4_w4a16: bool = False,
839 w1_scale: Optional[torch.Tensor] = None,
840 w2_scale: Optional[torch.Tensor] = None,
841 + w1_zp: Optional[torch.Tensor] = None,
842 + w2_zp: Optional[torch.Tensor] = None,
843 a1_scale: Optional[torch.Tensor] = None,
844 a2_scale: Optional[torch.Tensor] = None,
845 block_shape: Optional[List[int]] = None) -> None:
@@ -593,14 +862,18 @@ def outplace_fused_experts(
862 topk_ids: torch.Tensor,
863 use_fp8_w8a8: bool = False,
864 use_int8_w8a16: bool = False,
865 + use_int4_w4a16: bool = False,
866 w1_scale: Optional[torch.Tensor] = None,
867 w2_scale: Optional[torch.Tensor] = None,
868 + w1_zp: Optional[torch.Tensor] = None,
869 + w2_zp: Optional[torch.Tensor] = None,
870 a1_scale: Optional[torch.Tensor] = None,
871 a2_scale: Optional[torch.Tensor] = None,
872 block_shape: Optional[List[int]] = None) -> torch.Tensor:
873 return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
602 - False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
603 - w2_scale, a1_scale, a2_scale, block_shape)
874 + False, use_fp8_w8a8, use_int8_w8a16,
875 + use_int4_w4a16, w1_scale, w2_scale, w1_zp, w2_zp,
876 + a1_scale, a2_scale, block_shape)
877
878
879 def outplace_fused_experts_fake(
@@ -611,8 +884,11 @@ def outplace_fused_experts_fake(
884 topk_ids: torch.Tensor,
885 use_fp8_w8a8: bool = False,
886 use_int8_w8a16: bool = False,
887 + use_int4_w4a16: bool = False,
888 w1_scale: Optional[torch.Tensor] = None,
889 w2_scale: Optional[torch.Tensor] = None,
890 + w1_zp: Optional[torch.Tensor] = None,
891 + w2_zp: Optional[torch.Tensor] = None,
892 a1_scale: Optional[torch.Tensor] = None,
893 a2_scale: Optional[torch.Tensor] = None,
894 block_shape: Optional[List[int]] = None) -> torch.Tensor:
@@ -635,8 +911,11 @@ def fused_experts(hidden_states: torch.Tensor,
911 inplace: bool = False,
912 use_fp8_w8a8: bool = False,
913 use_int8_w8a16: bool = False,
914 + use_int4_w4a16: bool = False,
915 w1_scale: Optional[torch.Tensor] = None,
916 w2_scale: Optional[torch.Tensor] = None,
917 + w1_zp: Optional[torch.Tensor] = None,
918 + w2_zp: Optional[torch.Tensor] = None,
919 a1_scale: Optional[torch.Tensor] = None,
920 a2_scale: Optional[torch.Tensor] = None,
921 block_shape: Optional[List[int]] = None):
@@ -644,16 +923,15 @@ def fused_experts(hidden_states: torch.Tensor,
923 torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
924 topk_weights, topk_ids,
925 use_fp8_w8a8, use_int8_w8a16,
647 - w1_scale, w2_scale, a1_scale,
926 + use_int4_w4a16, w1_scale,
927 + w2_scale, w1_zp, w2_zp, a1_scale,
928 a2_scale, block_shape)
929 return hidden_states
930 else:
651 - return torch.ops.vllm.outplace_fused_experts(hidden_states, w1, w2,
652 - topk_weights, topk_ids,
653 - use_fp8_w8a8,
654 - use_int8_w8a16, w1_scale,
655 - w2_scale, a1_scale,
656 - a2_scale, block_shape)
931 + return torch.ops.vllm.outplace_fused_experts(
932 + hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
933 + use_int8_w8a16, use_int4_w4a16, w1_scale, w2_scale, w1_zp, w2_zp,
934 + a1_scale, a2_scale, block_shape)
935
936
937 def fused_experts_impl(hidden_states: torch.Tensor,
@@ -664,13 +942,21 @@ def fused_experts_impl(hidden_states: torch.Tensor,
942 inplace: bool = False,
943 use_fp8_w8a8: bool = False,
944 use_int8_w8a16: bool = False,
945 + use_int4_w4a16: bool = False,
946 w1_scale: Optional[torch.Tensor] = None,
947 w2_scale: Optional[torch.Tensor] = None,
948 + w1_zp: Optional[torch.Tensor] = None,
949 + w2_zp: Optional[torch.Tensor] = None,
950 a1_scale: Optional[torch.Tensor] = None,
951 a2_scale: Optional[torch.Tensor] = None,
952 block_shape: Optional[List[int]] = None):
953 # Check constraints.
673 - assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
954 + if use_int4_w4a16:
955 + assert hidden_states.shape[1] // 2 == w1.shape[
956 + 2], "Hidden size mismatch"
957 + else:
958 + assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
959 +
960 assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
961 assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
962 assert w1.is_contiguous(), "Expert weights1 must be contiguous"
@@ -687,6 +973,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
973 M = min(num_tokens, CHUNK_SIZE)
974 config_dtype = get_config_dtype_str(use_fp8_w8a8=use_fp8_w8a8,
975 use_int8_w8a16=use_int8_w8a16,
976 + use_int4_w4a16=use_int4_w4a16,
977 dtype=hidden_states.dtype)
978
979 get_config_func = functools.partial(
@@ -755,6 +1042,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1042 intermediate_cache1,
1043 a1_scale,
1044 w1_scale,
1045 + w1_zp,
1046 curr_topk_weights,
1047 curr_topk_ids,
1048 sorted_token_ids,
@@ -766,6 +1054,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1054 compute_type=compute_type,
1055 use_fp8_w8a8=use_fp8_w8a8,
1056 use_int8_w8a16=use_int8_w8a16,
1057 + use_int4_w4a16=use_int4_w4a16,
1058 block_shape=block_shape)
1059
1060 torch.ops._C.silu_and_mul(intermediate_cache2,
@@ -776,6 +1065,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1065 intermediate_cache3,
1066 a2_scale,
1067 w2_scale,
1068 + w2_zp,
1069 curr_topk_weights,
1070 curr_topk_ids,
1071 sorted_token_ids,
@@ -787,6 +1077,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
1077 compute_type=compute_type,
1078 use_fp8_w8a8=use_fp8_w8a8,
1079 use_int8_w8a16=use_int8_w8a16,
1080 + use_int4_w4a16=use_int4_w4a16,
1081 block_shape=block_shape)
1082
1083 ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
@@ -808,8 +1099,11 @@ def fused_moe(
1099 custom_routing_function: Optional[Callable] = None,
1100 use_fp8_w8a8: bool = False,
1101 use_int8_w8a16: bool = False,
1102 + use_int4_w4a16: bool = False,
1103 w1_scale: Optional[torch.Tensor] = None,
1104 w2_scale: Optional[torch.Tensor] = None,
1105 + w1_zp: Optional[torch.Tensor] = None,
1106 + w2_zp: Optional[torch.Tensor] = None,
1107 a1_scale: Optional[torch.Tensor] = None,
1108 a2_scale: Optional[torch.Tensor] = None,
1109 block_shape: Optional[List[int]] = None,
@@ -834,8 +1128,12 @@ def fused_moe(
1128 note: Deepseekv2 model uses grouped_topk
1129 - use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner
1130 products for w1 and w2. Defaults to False.
837 - - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner
838 - products for w1 and w2. Defaults to False.
1131 + - use_int8_w8a16 (bool): If True, use matmul of int8 weight and bf16/fp16
1132 + activation to compute the inner products for w1 and w2.
1133 + Defaults to False.
1134 + - use_int4_w4a16 (bool): If True, use matmul of int4 weight and bf16/fp16
1135 + activation to compute the inner products for w1 and w2.
1136 + Defaults to False.
1137 - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
1138 w1.
1139 - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
@@ -873,8 +1171,11 @@ def fused_moe(
1171 inplace=inplace,
1172 use_fp8_w8a8=use_fp8_w8a8,
1173 use_int8_w8a16=use_int8_w8a16,
1174 + use_int4_w4a16=use_int4_w4a16,
1175 w1_scale=w1_scale,
1176 w2_scale=w2_scale,
1177 + w1_zp=w1_zp,
1178 + w2_zp=w2_zp,
1179 a1_scale=a1_scale,
1180 a2_scale=a2_scale,
1181 block_shape=block_shape)
vllm/model_executor/layers/quantization/__init__.py
+5 -2
@@ -26,7 +26,8 @@ QUANTIZATION_METHODS: List[str] = [
26 "experts_int8",
27 "neuron_quant",
28 "ipex",
29 - "quark"
29 + "quark",
30 + "moe_wna16"
31 ]
32
33 # The customized quantization methods which will be added to this dict.
@@ -94,6 +95,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
95 from .ipex_quant import IPEXConfig
96 from .marlin import MarlinConfig
97 from .modelopt import ModelOptFp8Config
98 + from .moe_wna16 import MoeWNA16Config
99 from .neuron_quant import NeuronQuantConfig
100 from .qqq import QQQConfig
101 from .tpu_int8 import Int8TpuConfig
@@ -121,7 +123,8 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
123 "experts_int8": ExpertsInt8Config,
124 "neuron_quant": NeuronQuantConfig,
125 "ipex": IPEXConfig,
124 - "quark": QuarkConfig
126 + "quark": QuarkConfig,
127 + "moe_wna16": MoeWNA16Config,
128 }
129 # Update the `method_to_config` with customized quantization methods.
130 method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
vllm/model_executor/layers/quantization/moe_wna16.py new
+424
@@ -0,0 +1,424 @@
1 +from typing import Any, Callable, Dict, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
6 +from vllm.model_executor.layers.fused_moe.layer import (
7 + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
8 +from vllm.model_executor.layers.linear import UnquantizedLinearMethod
9 +from vllm.model_executor.layers.quantization.awq import (AWQConfig,
10 + AWQLinearMethod)
11 +from vllm.model_executor.layers.quantization.awq_marlin import (
12 + AWQMarlinConfig, AWQMarlinLinearMethod)
13 +from vllm.model_executor.layers.quantization.base_config import (
14 + QuantizationConfig, QuantizeMethodBase)
15 +from vllm.model_executor.layers.quantization.gptq import (GPTQConfig,
16 + GPTQLinearMethod)
17 +from vllm.model_executor.layers.quantization.gptq_marlin import (
18 + GPTQMarlinConfig, GPTQMarlinLinearMethod)
19 +from vllm.model_executor.utils import set_weight_attrs
20 +from vllm.platforms import current_platform
21 +
22 +
23 +class MoeWNA16Config(QuantizationConfig):
24 + """Config class for MOE WNA16 (W8A16/W4A16) quantization."""
25 +
26 + def __init__(self, linear_quant_method: str, weight_bits: int,
27 + group_size: int, has_zp: bool, lm_head_quantized: bool,
28 + modules_to_not_convert: Optional[List[str]],
29 + full_config: Dict[str, Any]) -> None:
30 + self.weight_bits = weight_bits
31 + self.group_size = group_size
32 + self.has_zp = has_zp
33 + self.bit8_pack_factor = 8 // self.weight_bits
34 + self.lm_head_quantized = lm_head_quantized
35 + self.linear_quant_method = linear_quant_method
36 + self.full_config = full_config
37 + self.use_marlin = False
38 + if self.linear_quant_method == "gptq":
39 + self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(
40 + full_config)
41 + elif self.linear_quant_method == "awq":
42 + capability_tuple = current_platform.get_device_capability()
43 + device_capability = (-1 if capability_tuple is None else
44 + capability_tuple.to_int())
45 + awq_min_capability = AWQConfig.get_min_capability()
46 + if device_capability < awq_min_capability:
47 + raise ValueError(
48 + "The quantization method moe_wna16 + awq is not supported "
49 + "for the current GPU. "
50 + f"Minimum capability: {awq_min_capability}. "
51 + f"Current capability: {device_capability}.")
52 + self.use_marlin = AWQMarlinConfig.is_awq_marlin_compatible(
53 + full_config)
54 + else:
55 + raise ValueError("moe_wna16 only support gptq and awq.")
56 +
57 + if modules_to_not_convert is None:
58 + self.modules_to_not_convert = []
59 + else:
60 + self.modules_to_not_convert = modules_to_not_convert
61 +
62 + @classmethod
63 + def get_name(cls) -> str:
64 + return "moe_wna16"
65 +
66 + @classmethod
67 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
68 + return [torch.bfloat16, torch.half]
69 +
70 + @classmethod
71 + def get_min_capability(cls) -> int:
72 + return 70
73 +
74 + @classmethod
75 + def get_config_filenames(cls) -> List[str]:
76 + return ["quantize_config.json"]
77 +
78 + @classmethod
79 + def from_config(cls, config: Dict[str, Any]) -> "MoeWNA16Config":
80 + linear_quant_method = cls.get_from_keys(config, ["quant_method"])
81 + weight_bits = cls.get_from_keys(config, ["bits"])
82 + group_size = cls.get_from_keys(config, ["group_size"])
83 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
84 + default=False)
85 + if linear_quant_method == "gptq":
86 + has_zp = not cls.get_from_keys(config, ["sym"])
87 + modules_to_not_convert = []
88 + elif linear_quant_method == "awq":
89 + has_zp = cls.get_from_keys(config, ["zero_point"])
90 + modules_to_not_convert = cls.get_from_keys(
91 + config, ["modules_to_not_convert"])
92 + else:
93 + raise ValueError("moe_wna16 only support gptq and awq.")
94 +
95 + return cls(linear_quant_method, weight_bits, group_size, has_zp,
96 + lm_head_quantized, modules_to_not_convert, config)
97 +
98 + @classmethod
99 + def override_quantization_method(cls, hf_quant_cfg,
100 + user_quant) -> Optional[str]:
101 + can_convert = cls.is_moe_wna16_compatible(hf_quant_cfg)
102 + if can_convert and user_quant == "moe_wna16":
103 + return cls.get_name()
104 + return None
105 +
106 + @classmethod
107 + def is_moe_wna16_compatible(cls, quant_config: Dict[str, Any]):
108 + # Extract data from quant config.
109 + quant_method = quant_config.get("quant_method", "").lower()
110 + num_bits = quant_config.get("bits")
111 + desc_act = quant_config.get("desc_act")
112 +
113 + capability_tuple = current_platform.get_device_capability()
114 + device_capability = (-1 if capability_tuple is None else
115 + capability_tuple.to_int())
116 + awq_min_capability = AWQConfig.get_min_capability()
117 +
118 + gptq_compatible = quant_method == "gptq" and \
119 + not desc_act and num_bits in [4, 8]
120 + awq_compatible = quant_method == "awq" and num_bits == 4 and \
121 + device_capability >= awq_min_capability
122 +
123 + return gptq_compatible or awq_compatible
124 +
125 + def get_quant_method(self, layer: torch.nn.Module,
126 + prefix: str) -> Optional["QuantizeMethodBase"]:
127 + if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
128 + return UnquantizedLinearMethod()
129 + elif isinstance(layer, FusedMoE):
130 + return MoeWNA16Method(self)
131 + else:
132 + if self.linear_quant_method == "gptq":
133 + if self.use_marlin:
134 + return GPTQMarlinLinearMethod(
135 + GPTQMarlinConfig.from_config(self.full_config))
136 + else:
137 + return GPTQLinearMethod(
138 + GPTQConfig.from_config(self.full_config))
139 + elif self.linear_quant_method == "awq":
140 + if self.use_marlin:
141 + return AWQMarlinLinearMethod(
142 + AWQMarlinConfig.from_config(self.full_config))
143 + else:
144 + return AWQLinearMethod(
145 + AWQConfig.from_config(self.full_config))
146 + else:
147 + raise ValueError("moe_wna16 only support gptq and awq.")
148 +
149 +
150 +def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]):
151 + return any(module_name in prefix for module_name in modules_to_not_convert)
152 +
153 +
154 +class MoeWNA16Method(FusedMoEMethodBase):
155 + """Linear method for MOE WNA16 (W8A16/W4A16) quantization.
156 +
157 + Args:
158 + quant_config: The MOE WNA16 (W8A16/W4A16) quantization config.
159 + """
160 +
161 + def __init__(self, quant_config: MoeWNA16Config):
162 + self.quant_config = quant_config
163 +
164 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
165 + hidden_size: int, intermediate_size_per_partition: int,
166 + params_dtype: torch.dtype, **extra_weight_attrs):
167 +
168 + layer.quant_config = self.quant_config
169 + bit8_pack_factor = self.quant_config.bit8_pack_factor
170 + group_size = self.quant_config.group_size
171 + group_size_div_factor = 1
172 +
173 + # make intermediate_size and hidden_size diviable by group_size
174 + # we reduce the group size to ensure that
175 + # and we would repeat the loaded_weight later
176 + while intermediate_size_per_partition % group_size or \
177 + hidden_size % group_size:
178 + group_size = group_size // 2
179 + group_size_div_factor *= 2
180 + assert group_size >= 32
181 + layer.group_size = group_size
182 + layer.group_size_div_factor = group_size_div_factor
183 +
184 + strategy = FusedMoeWeightScaleSupported.GROUP.value
185 + extra_weight_attrs.update({
186 + "quant_method": strategy,
187 + "is_transposed": False
188 + })
189 +
190 + assert 'weight_loader' in extra_weight_attrs
191 + weight_loader = extra_weight_attrs['weight_loader']
192 + wrapped_weight_loader = MoeWNA16Method.get_weight_loader(
193 + layer, weight_loader)
194 + extra_weight_attrs['weight_loader'] = wrapped_weight_loader
195 +
196 + # Fused gate_up_proj (column parallel)
197 + w13_qweight = torch.nn.Parameter(torch.empty(
198 + num_experts,
199 + 2 * intermediate_size_per_partition,
200 + hidden_size // bit8_pack_factor,
201 + dtype=torch.uint8),
202 + requires_grad=False)
203 + layer.register_parameter("w13_qweight", w13_qweight)
204 + set_weight_attrs(w13_qweight, extra_weight_attrs)
205 +
206 + # down_proj (row parallel)
207 + w2_qweight = torch.nn.Parameter(torch.empty(
208 + num_experts,
209 + hidden_size,
210 + intermediate_size_per_partition // bit8_pack_factor,
211 + dtype=torch.uint8),
212 + requires_grad=False)
213 + layer.register_parameter("w2_qweight", w2_qweight)
214 + set_weight_attrs(w2_qweight, extra_weight_attrs)
215 +
216 + w13_scales = torch.nn.Parameter(torch.zeros(
217 + num_experts,
218 + 2 * intermediate_size_per_partition,
219 + hidden_size // group_size,
220 + dtype=params_dtype),
221 + requires_grad=False)
222 + layer.register_parameter("w13_scales", w13_scales)
223 + set_weight_attrs(w13_scales, extra_weight_attrs)
224 +
225 + w2_scales = torch.nn.Parameter(torch.zeros(
226 + num_experts,
227 + hidden_size,
228 + intermediate_size_per_partition // group_size,
229 + dtype=params_dtype),
230 + requires_grad=False)
231 + layer.register_parameter("w2_scales", w2_scales)
232 + set_weight_attrs(w2_scales, extra_weight_attrs)
233 +
234 + if self.quant_config.has_zp:
235 + w13_qzeros = torch.nn.Parameter(torch.zeros(
236 + num_experts,
237 + 2 * intermediate_size_per_partition // bit8_pack_factor,
238 + hidden_size // group_size,
239 + dtype=torch.uint8),
240 + requires_grad=False)
241 + layer.register_parameter("w13_qzeros", w13_qzeros)
242 + set_weight_attrs(w13_qzeros, extra_weight_attrs)
243 +
244 + w2_qzeros = torch.nn.Parameter(torch.zeros(
245 + num_experts,
246 + hidden_size // bit8_pack_factor,
247 + intermediate_size_per_partition // group_size,
248 + dtype=torch.uint8),
249 + requires_grad=False)
250 + layer.register_parameter("w2_qzeros", w2_qzeros)
251 + set_weight_attrs(w2_qzeros, extra_weight_attrs)
252 +
253 + if self.quant_config.linear_quant_method == "gptq":
254 + # some param are unused, but we need to init them in order to
255 + # load weights
256 + invalid_param_keys = ["w13_g_idx", "w2_g_idx"]
257 + if not self.quant_config.has_zp:
258 + invalid_param_keys += ["w13_qzeros", "w2_qzeros"]
259 + for key in invalid_param_keys:
260 + param = torch.nn.Parameter(torch.empty((0, ),
261 + dtype=torch.int32),
262 + requires_grad=False)
263 + layer.register_parameter(key, param)
264 + set_weight_attrs(param, extra_weight_attrs)
265 +
266 + def apply(
267 + self,
268 + layer: torch.nn.Module,
269 + x: torch.Tensor,
270 + router_logits: torch.Tensor,
271 + top_k: int,
272 + renormalize: bool,
273 + use_grouped_topk: bool = False,
274 + topk_group: Optional[int] = None,
275 + num_expert_group: Optional[int] = None,
276 + custom_routing_function: Optional[Callable] = None,
277 + scoring_func: str = "softmax",
278 + e_score_correction_bias: Optional[torch.Tensor] = None,
279 + ) -> torch.Tensor:
280 + from vllm.model_executor.layers.fused_moe import fused_experts
281 +
282 + topk_weights, topk_ids = FusedMoE.select_experts(
283 + hidden_states=x,
284 + router_logits=router_logits,
285 + use_grouped_topk=use_grouped_topk,
286 + top_k=top_k,
287 + renormalize=renormalize,
288 + topk_group=topk_group,
289 + num_expert_group=num_expert_group,
290 + custom_routing_function=custom_routing_function,
291 + scoring_func=scoring_func,
292 + e_score_correction_bias=e_score_correction_bias)
293 +
294 + weight_bits = self.quant_config.weight_bits
295 + has_zp = self.quant_config.has_zp
296 +
297 + return fused_experts(x,
298 + layer.w13_qweight,
299 + layer.w2_qweight,
300 + topk_weights=topk_weights,
301 + topk_ids=topk_ids,
302 + inplace=True,
303 + use_int4_w4a16=weight_bits == 4,
304 + use_int8_w8a16=weight_bits == 8,
305 + w1_scale=layer.w13_scales,
306 + w2_scale=layer.w2_scales,
307 + w1_zp=layer.w13_qzeros if has_zp else None,
308 + w2_zp=layer.w2_qzeros if has_zp else None,
309 + block_shape=[0, layer.group_size])
310 +
311 + @staticmethod
312 + def get_weight_loader(layer, weight_loader):
313 +
314 + def convert_awq_tensor(tensor, tensor_type):
315 + # convert awq qweight/qzeros to a standard format (assume int4)
316 + # qweight: (k, n // pack_factor_bit32) -> (n, k // pack_factor_bit8)
317 + # qzeros: (k // group_size, n // pack_factor_bit32) ->
318 + # (n // pack_factor_bit8, k // group_size)
319 + # pack_factor_bit32 = 32 // weight_bits
320 + # pack_factor_bit8 = 8 // weight_bits
321 +
322 + # 0. suppose origin shape (a, b), dtype int32
323 + # 1. convert to uint8, shape (a, b) -> (a, 4 * b)
324 + size0 = tensor.size(0)
325 + tensor = tensor.view(torch.uint8)
326 +
327 + # 2. unpack to uint4 (only when weight_bits == 4)
328 + # shape (a, 4 * b) -> (a, 4 * b, 2)
329 + shifter = torch.tensor([0, 4],
330 + dtype=torch.uint8,
331 + device=tensor.device)
332 + tensor = (tensor[:, :, None] >> shifter) & 0xF
333 +
334 + # 3. change order, see
335 + # https://github.com/casper-hansen/AutoAWQ/blob/v0.2.8/awq/utils/quant_utils.py
336 + # shape -> (a, 4 * b * pack_factor_bit8)
337 + reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
338 + tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
339 + tensor = tensor.view(size0, -1)
340 +
341 + # 4. transpose, shape -> (4 * b * pack_factor_bit8, a)
342 + tensor = tensor.T.contiguous()
343 +
344 + # 5. repack (only when weight_bits == 4)
345 + # qweight shape -> (4 * b * pack_factor_bit8, a // pack_factor_bit8)
346 + # qzeros shape -> (4 * b, a)
347 +
348 + if tensor_type == "qweight":
349 + tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
350 + elif tensor_type == "qzeros":
351 + tensor = tensor[1::2, :] * 16 + tensor[::2, :]
352 + return tensor
353 +
354 + def convert_gptq_int4_qzeros(tensor):
355 + tensor = tensor.view(torch.uint8)
356 + shifter = torch.tensor([0, 4],
357 + dtype=torch.uint8,
358 + device=tensor.device)
359 + tensor = (tensor[:, :, None] >> shifter) & 0xF
360 + tensor = tensor + 1
361 + tensor = tensor[:, :, 0] + tensor[:, :, 1] * 16
362 + return tensor
363 +
364 + def moe_wna16_weight_loader(param: torch.nn.Parameter,
365 + loaded_weight: torch.Tensor,
366 + weight_name: str, shard_id: str,
367 + expert_id: int):
368 + if "g_idx" in weight_name:
369 + return
370 + if not layer.quant_config.has_zp and "qzeros" in weight_name:
371 + return
372 +
373 + device = get_tp_group().device
374 + tp_rank = get_tensor_model_parallel_rank()
375 + loaded_weight = loaded_weight.to(device)
376 + shard_size = layer.intermediate_size_per_partition
377 +
378 + # convert gptq and awq weight to a standard format
379 + if layer.quant_config.linear_quant_method == "awq":
380 + assert layer.quant_config.weight_bits == 4
381 + if "weight" in weight_name:
382 + loaded_weight = convert_awq_tensor(loaded_weight,
383 + "qweight")
384 + elif "zeros" in weight_name:
385 + loaded_weight = convert_awq_tensor(loaded_weight, "qzeros")
386 + else:
387 + loaded_weight = loaded_weight.T
388 + elif layer.quant_config.linear_quant_method == "gptq":
389 + assert layer.quant_config.weight_bits in [4, 8]
390 + if "weight" in weight_name:
391 + loaded_weight = loaded_weight.T.contiguous().view(
392 + torch.uint8)
393 + elif "zeros" in weight_name:
394 + # add 1 to gptq qzeros to align with awq
395 + loaded_weight = loaded_weight.view(torch.uint8)
396 + if layer.quant_config.weight_bits == 4:
397 + loaded_weight = convert_gptq_int4_qzeros(
398 + loaded_weight).T
399 + else:
400 + loaded_weight = loaded_weight.T + 1
401 + else:
402 + loaded_weight = loaded_weight.T
403 +
404 + # repeat the qzeros/scales to fit new group size
405 + if layer.group_size_div_factor > 1 and \
406 + "qzeros" in weight_name or "scales" in weight_name:
407 + loaded_weight = loaded_weight.repeat_interleave(
408 + layer.group_size_div_factor, 1)
409 +
410 + if "w13_qzeros" in weight_name:
411 + tensor = loaded_weight.view(layer.tp_size, -1,
412 + loaded_weight.size(1))[tp_rank]
413 + if shard_id == "w1":
414 + param.data[expert_id, :shard_size // 2] = tensor
415 + else:
416 + param.data[expert_id, shard_size // 2:] = tensor
417 + elif "w2_qzeros" in weight_name:
418 + param.data[expert_id] = loaded_weight.view(
419 + loaded_weight.size(0), layer.tp_size, -1)[:, tp_rank]
420 + else:
421 + weight_loader(param, loaded_weight, weight_name, shard_id,
422 + expert_id)
423 +
424 + return moe_wna16_weight_loader