A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix Marlin MoE act order when is_k_full == False (#8741)
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
ElizaWszola committed
Sep 29, 2024 at 03:19 UTC
d081da0064b5cda9e344f0fd519d67523a437a39
4 files changed
+37
-18
csrc/core/exception.hpp
new
+3
@@ -0,0 +1,3 @@
1
+#pragma once
2
+
3
+#define VLLM_IMPLIES(p, q) (!(p) || (q))
csrc/moe/marlin_moe_ops.cu
+6
-6
@@ -25,6 +25,7 @@
25
26
#include <iostream>
27
28
+#include "core/exception.hpp"
29
#include "core/scalar_type.hpp"
30
#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
31
#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
@@ -189,7 +190,7 @@ int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
190
int load_groups =
191
tb_groups * STAGES * 2; // Chunk size is 2x pipeline over dim K
192
load_groups = max(load_groups, 32); // We load at least 32 scale groups
192
- return load_groups * tb_n * 2;
193
+ return load_groups * tb_n * 4;
194
195
} else {
196
int tb_scales = tb_groups * tb_n * 2;
@@ -433,11 +434,7 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
434
int4* C_ptr = (int4*)C;
435
const float* topk_weights_ptr = (const float*)topk_weights;
436
const int* sorted_ids_ptr = (const int*)sorted_ids;
436
- const int4* s_ptr =
437
- (const int4*)s +
438
- (((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
439
- prob_n / 8) *
440
- expert_idx;
437
+ const int4* s_ptr = (const int4*)s + num_groups * prob_n / 8 * expert_idx;
438
const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
439
const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
440
int* locks = (int*)workspace;
@@ -521,6 +518,9 @@ torch::Tensor marlin_gemm_moe(
518
" is not size_n = ", size_n);
519
num_groups = b_scales.size(1);
520
521
+ TORCH_CHECK(VLLM_IMPLIES(!is_k_full, has_act_order),
522
+ "if is_k_full is false, has_act_order must be true");
523
+
524
if (has_act_order) {
525
if (is_k_full) {
526
TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
tests/kernels/test_moe.py
+23
-9
@@ -145,6 +145,7 @@ def compute_max_diff(output, output_ref):
145
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
146
@pytest.mark.parametrize("act_order", [True, False])
147
@pytest.mark.parametrize("num_bits", [4, 8])
148
+@pytest.mark.parametrize("is_k_full", [True, False])
149
def test_fused_marlin_moe(
150
m: int,
151
n: int,
@@ -154,6 +155,7 @@ def test_fused_marlin_moe(
155
group_size: int,
156
act_order: bool,
157
num_bits: int,
158
+ is_k_full: bool,
159
):
160
seed_everything(7)
161
@@ -166,6 +168,9 @@ def test_fused_marlin_moe(
168
return
169
if group_size in (k, n):
170
return
171
+ else:
172
+ if not is_k_full:
173
+ return
174
175
quant_type = (scalar_types.uint4b8
176
if num_bits == 4 else scalar_types.uint8b128)
@@ -246,6 +251,7 @@ def test_fused_marlin_moe(
251
w1_scale=scales1,
252
w2_scale=scales2,
253
num_bits=num_bits,
254
+ is_k_full=is_k_full,
255
)
256
257
assert compute_max_diff(marlin_output, triton_output) < 4e-2
@@ -290,6 +296,7 @@ def test_fused_marlin_moe(
296
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
297
@pytest.mark.parametrize("act_order", [True, False])
298
@pytest.mark.parametrize("num_bits", [4, 8])
299
+@pytest.mark.parametrize("is_k_full", [True, False])
300
def test_single_marlin_moe_multiply(
301
m: int,
302
n: int,
@@ -299,6 +306,7 @@ def test_single_marlin_moe_multiply(
306
group_size: int,
307
act_order: bool,
308
num_bits: int,
309
+ is_k_full: bool,
310
):
311
if topk > e:
312
return
@@ -309,6 +317,9 @@ def test_single_marlin_moe_multiply(
317
return
318
if group_size == k:
319
return
320
+ else:
321
+ if not is_k_full:
322
+ return
323
324
quant_type = (scalar_types.uint4b8
325
if num_bits == 4 else scalar_types.uint8b128)
@@ -339,15 +350,18 @@ def test_single_marlin_moe_multiply(
350
sort_indices = stack_and_dev(sort_indices_l)
351
352
score = torch.randn((m, e), device="cuda", dtype=dtype)
342
- marlin_output = single_marlin_moe(a,
343
- qweight,
344
- scales,
345
- score,
346
- g_idx,
347
- sort_indices,
348
- topk,
349
- renormalize=False,
350
- num_bits=num_bits)
353
+ marlin_output = single_marlin_moe(
354
+ a,
355
+ qweight,
356
+ scales,
357
+ score,
358
+ g_idx,
359
+ sort_indices,
360
+ topk,
361
+ renormalize=False,
362
+ num_bits=num_bits,
363
+ is_k_full=is_k_full,
364
+ )
365
torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
366
367
assert compute_max_diff(marlin_output, torch_output) < 1e-2
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+5
-3
@@ -21,6 +21,7 @@ def single_marlin_moe(
21
renormalize: bool,
22
override_config: Optional[Dict[str, Any]] = None,
23
num_bits: int = 8,
24
+ is_k_full: bool = True,
25
) -> torch.Tensor:
26
"""
27
This function computes the multiplication of hidden_states with expert
@@ -86,7 +87,7 @@ def single_marlin_moe(
87
88
intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
89
hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
89
- g_idx, perm, workspace, scalar_type, M, N, K, True, E, topk,
90
+ g_idx, perm, workspace, scalar_type, M, N, K, is_k_full, E, topk,
91
block_size_m, True, False)
92
93
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
@@ -107,6 +108,7 @@ def fused_marlin_moe(
108
w1_scale: Optional[torch.Tensor] = None,
109
w2_scale: Optional[torch.Tensor] = None,
110
num_bits: int = 8,
111
+ is_k_full: bool = True,
112
) -> torch.Tensor:
113
"""
114
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -199,7 +201,7 @@ def fused_marlin_moe(
201
M,
202
2 * N,
203
K,
202
- True,
204
+ is_k_full,
205
E,
206
topk,
207
block_size_m,
@@ -223,7 +225,7 @@ def fused_marlin_moe(
225
M,
226
K,
227
N,
226
- True,
228
+ is_k_full,
229
E,
230
topk,
231
block_size_m,