@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d081da0064
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix Marlin MoE act order when is_k_full == False (#8741)

Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>

ElizaWszola committed Sep 29, 2024 at 03:19 UTC d081da0064b5cda9e344f0fd519d67523a437a39
4 files changed +37 -18
csrc/core/exception.hpp new
+3
@@ -0,0 +1,3 @@
1 +#pragma once
2 +
3 +#define VLLM_IMPLIES(p, q) (!(p) || (q))
csrc/moe/marlin_moe_ops.cu
+6 -6
@@ -25,6 +25,7 @@
25
26 #include <iostream>
27
28 +#include "core/exception.hpp"
29 #include "core/scalar_type.hpp"
30 #include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
31 #include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
@@ -189,7 +190,7 @@ int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
190 int load_groups =
191 tb_groups * STAGES * 2; // Chunk size is 2x pipeline over dim K
192 load_groups = max(load_groups, 32); // We load at least 32 scale groups
192 - return load_groups * tb_n * 2;
193 + return load_groups * tb_n * 4;
194
195 } else {
196 int tb_scales = tb_groups * tb_n * 2;
@@ -433,11 +434,7 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
434 int4* C_ptr = (int4*)C;
435 const float* topk_weights_ptr = (const float*)topk_weights;
436 const int* sorted_ids_ptr = (const int*)sorted_ids;
436 - const int4* s_ptr =
437 - (const int4*)s +
438 - (((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
439 - prob_n / 8) *
440 - expert_idx;
437 + const int4* s_ptr = (const int4*)s + num_groups * prob_n / 8 * expert_idx;
438 const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
439 const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
440 int* locks = (int*)workspace;
@@ -521,6 +518,9 @@ torch::Tensor marlin_gemm_moe(
518 " is not size_n = ", size_n);
519 num_groups = b_scales.size(1);
520
521 + TORCH_CHECK(VLLM_IMPLIES(!is_k_full, has_act_order),
522 + "if is_k_full is false, has_act_order must be true");
523 +
524 if (has_act_order) {
525 if (is_k_full) {
526 TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
tests/kernels/test_moe.py
+23 -9
@@ -145,6 +145,7 @@ def compute_max_diff(output, output_ref):
145 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
146 @pytest.mark.parametrize("act_order", [True, False])
147 @pytest.mark.parametrize("num_bits", [4, 8])
148 +@pytest.mark.parametrize("is_k_full", [True, False])
149 def test_fused_marlin_moe(
150 m: int,
151 n: int,
@@ -154,6 +155,7 @@ def test_fused_marlin_moe(
155 group_size: int,
156 act_order: bool,
157 num_bits: int,
158 + is_k_full: bool,
159 ):
160 seed_everything(7)
161
@@ -166,6 +168,9 @@ def test_fused_marlin_moe(
168 return
169 if group_size in (k, n):
170 return
171 + else:
172 + if not is_k_full:
173 + return
174
175 quant_type = (scalar_types.uint4b8
176 if num_bits == 4 else scalar_types.uint8b128)
@@ -246,6 +251,7 @@ def test_fused_marlin_moe(
251 w1_scale=scales1,
252 w2_scale=scales2,
253 num_bits=num_bits,
254 + is_k_full=is_k_full,
255 )
256
257 assert compute_max_diff(marlin_output, triton_output) < 4e-2
@@ -290,6 +296,7 @@ def test_fused_marlin_moe(
296 @pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
297 @pytest.mark.parametrize("act_order", [True, False])
298 @pytest.mark.parametrize("num_bits", [4, 8])
299 +@pytest.mark.parametrize("is_k_full", [True, False])
300 def test_single_marlin_moe_multiply(
301 m: int,
302 n: int,
@@ -299,6 +306,7 @@ def test_single_marlin_moe_multiply(
306 group_size: int,
307 act_order: bool,
308 num_bits: int,
309 + is_k_full: bool,
310 ):
311 if topk > e:
312 return
@@ -309,6 +317,9 @@ def test_single_marlin_moe_multiply(
317 return
318 if group_size == k:
319 return
320 + else:
321 + if not is_k_full:
322 + return
323
324 quant_type = (scalar_types.uint4b8
325 if num_bits == 4 else scalar_types.uint8b128)
@@ -339,15 +350,18 @@ def test_single_marlin_moe_multiply(
350 sort_indices = stack_and_dev(sort_indices_l)
351
352 score = torch.randn((m, e), device="cuda", dtype=dtype)
342 - marlin_output = single_marlin_moe(a,
343 - qweight,
344 - scales,
345 - score,
346 - g_idx,
347 - sort_indices,
348 - topk,
349 - renormalize=False,
350 - num_bits=num_bits)
353 + marlin_output = single_marlin_moe(
354 + a,
355 + qweight,
356 + scales,
357 + score,
358 + g_idx,
359 + sort_indices,
360 + topk,
361 + renormalize=False,
362 + num_bits=num_bits,
363 + is_k_full=is_k_full,
364 + )
365 torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
366
367 assert compute_max_diff(marlin_output, torch_output) < 1e-2
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+5 -3
@@ -21,6 +21,7 @@ def single_marlin_moe(
21 renormalize: bool,
22 override_config: Optional[Dict[str, Any]] = None,
23 num_bits: int = 8,
24 + is_k_full: bool = True,
25 ) -> torch.Tensor:
26 """
27 This function computes the multiplication of hidden_states with expert
@@ -86,7 +87,7 @@ def single_marlin_moe(
87
88 intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
89 hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
89 - g_idx, perm, workspace, scalar_type, M, N, K, True, E, topk,
90 + g_idx, perm, workspace, scalar_type, M, N, K, is_k_full, E, topk,
91 block_size_m, True, False)
92
93 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
@@ -107,6 +108,7 @@ def fused_marlin_moe(
108 w1_scale: Optional[torch.Tensor] = None,
109 w2_scale: Optional[torch.Tensor] = None,
110 num_bits: int = 8,
111 + is_k_full: bool = True,
112 ) -> torch.Tensor:
113 """
114 This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -199,7 +201,7 @@ def fused_marlin_moe(
201 M,
202 2 * N,
203 K,
202 - True,
204 + is_k_full,
205 E,
206 topk,
207 block_size_m,
@@ -223,7 +225,7 @@ def fused_marlin_moe(
225 M,
226 K,
227 N,
226 - True,
228 + is_k_full,
229 E,
230 topk,
231 block_size_m,