A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] adding chunking mechanism to fused_moe to handle large inputs (#6029)
Avshalom Manevich committed
Jul 2, 2024 at 00:08 UTC
12a59959ed3a78a50f9b48b0ec6ccd6c862b22b4
3 files changed
+74
-48
tests/kernels/test_moe.py
+1
-1
@@ -29,7 +29,7 @@ def torch_moe(a, w1, w2, score, topk):
29
topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
30
31
32
-@pytest.mark.parametrize("m", [512, 222, 33, 1])
32
+@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
33
@pytest.mark.parametrize("n", [2048, 256, 1024])
34
@pytest.mark.parametrize("k", [128, 511, 1024])
35
@pytest.mark.parametrize("e", [8, 64])
vllm/envs.py
+3
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32
VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
33
VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
34
VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
35
+ VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
36
VLLM_USE_RAY_COMPILED_DAG: bool = False
37
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
38
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -248,6 +249,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
249
# Only used for XLA devices such as TPUs.
250
"VLLM_XLA_CACHE_PATH":
251
lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
252
+ "VLLM_FUSED_MOE_CHUNK_SIZE":
253
+ lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
254
}
255
256
# end-env-vars-definition
vllm/model_executor/layers/fused_moe/fused_moe.py
+70
-47
@@ -8,6 +8,7 @@ import torch
8
import triton
9
import triton.language as tl
10
11
+import vllm.envs as envs
12
from vllm import _custom_ops as ops
13
from vllm.logger import init_logger
14
@@ -420,13 +421,12 @@ def fused_experts(hidden_states: torch.Tensor,
421
torch.float32, torch.float16, torch.bfloat16
422
]
423
423
- M, _ = hidden_states.shape
424
+ num_tokens, _ = hidden_states.shape
425
E, N, _ = w1.shape
425
-
426
- if M > 65536:
427
- # https://github.com/vllm-project/vllm/issues/5938
428
- raise ValueError("MoE kernel does not support more than 65536 tokens, "
429
- f"but got {M}")
426
+ # We execute the fused_moe kernel in chunks to circumvent this issue:
427
+ # https://github.com/vllm-project/vllm/issues/5938
428
+ CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE
429
+ M = min(num_tokens, CHUNK_SIZE)
430
431
if override_config:
432
config = override_config
@@ -455,51 +455,74 @@ def fused_experts(hidden_states: torch.Tensor,
455
device=hidden_states.device,
456
dtype=hidden_states.dtype)
457
458
- sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
459
- topk_ids, config['BLOCK_SIZE_M'], E)
458
compute_type = (tl.bfloat16
459
if hidden_states.dtype == torch.bfloat16 else tl.float16)
460
463
- invoke_fused_moe_kernel(hidden_states,
464
- w1,
465
- intermediate_cache1,
466
- a1_scale,
467
- w1_scale,
468
- topk_weights,
469
- topk_ids,
470
- sorted_token_ids,
471
- expert_ids,
472
- num_tokens_post_padded,
473
- False,
474
- topk_ids.shape[1],
475
- config,
476
- compute_type=compute_type,
477
- use_fp8=use_fp8)
478
-
479
- ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
480
-
481
- invoke_fused_moe_kernel(intermediate_cache2,
482
- w2,
483
- intermediate_cache3,
484
- a2_scale,
485
- w2_scale,
486
- topk_weights,
487
- topk_ids,
488
- sorted_token_ids,
489
- expert_ids,
490
- num_tokens_post_padded,
491
- True,
492
- 1,
493
- config,
494
- compute_type=compute_type,
495
- use_fp8=use_fp8)
496
-
461
if inplace:
498
- return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
499
- dim=1,
500
- out=hidden_states)
501
- return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
502
- dim=1)
462
+ out_hidden_states = hidden_states
463
+ else:
464
+ out_hidden_states = torch.empty_like(hidden_states)
465
+
466
+ for chunk in range((num_tokens // CHUNK_SIZE) + 1):
467
+ begin_chunk_idx, end_chunk_idx = (chunk * CHUNK_SIZE,
468
+ min((chunk + 1) * CHUNK_SIZE,
469
+ num_tokens))
470
+ curr_hidden_states = hidden_states[begin_chunk_idx:end_chunk_idx]
471
+ tokens_in_chunk, _ = curr_hidden_states.shape
472
+
473
+ if tokens_in_chunk == 0:
474
+ break
475
+
476
+ if tokens_in_chunk < CHUNK_SIZE:
477
+ # will only happen in the last chunk
478
+ intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
479
+ intermediate_cache2 = intermediate_cache2[:tokens_in_chunk]
480
+ intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
481
+
482
+ curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx]
483
+ curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx]
484
+
485
+ sorted_token_ids, expert_ids, num_tokens_post_padded = (
486
+ moe_align_block_size(curr_topk_ids, config['BLOCK_SIZE_M'], E))
487
+
488
+ invoke_fused_moe_kernel(curr_hidden_states,
489
+ w1,
490
+ intermediate_cache1,
491
+ a1_scale,
492
+ w1_scale,
493
+ curr_topk_weights,
494
+ curr_topk_ids,
495
+ sorted_token_ids,
496
+ expert_ids,
497
+ num_tokens_post_padded,
498
+ False,
499
+ topk_ids.shape[1],
500
+ config,
501
+ compute_type=compute_type,
502
+ use_fp8=use_fp8)
503
+
504
+ ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
505
+
506
+ invoke_fused_moe_kernel(intermediate_cache2,
507
+ w2,
508
+ intermediate_cache3,
509
+ a2_scale,
510
+ w2_scale,
511
+ curr_topk_weights,
512
+ curr_topk_ids,
513
+ sorted_token_ids,
514
+ expert_ids,
515
+ num_tokens_post_padded,
516
+ True,
517
+ 1,
518
+ config,
519
+ compute_type=compute_type,
520
+ use_fp8=use_fp8)
521
+
522
+ torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
523
+ dim=1,
524
+ out=out_hidden_states[begin_chunk_idx:end_chunk_idx])
525
+ return out_hidden_states
526
527
528
def fused_moe(