@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 145c2ff648
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Revert MoE Triton Config Default (#12629)

SUMMARY: * previous PR for pulling in block configs also changed defaults (https://github.com/vllm-project/vllm/pull/11589/files) for FP8 * this broke L4 MoE since there was not enough SHM for the default configuration * this reverts the non-block example to the default Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Robert Shaw committed Jan 31, 2025 at 18:28 UTC 145c2ff648ad0a300f880ac38811d0d8a2eb3e79
1 file changed +11 -30
vllm/model_executor/layers/fused_moe/fused_moe.py
+11 -30
@@ -660,36 +660,17 @@ def get_default_config(
660 is_marlin: bool,
661 block_shape: Optional[List[int]] = None,
662 ) -> Dict[str, int]:
663 - if dtype == "fp8_w8a8":
664 - if block_shape is None:
665 - config = {
666 - "BLOCK_SIZE_M": 128,
667 - "BLOCK_SIZE_N": 256,
668 - "BLOCK_SIZE_K": 128,
669 - "GROUP_SIZE_M": 32,
670 - "num_warps": 8,
671 - "num_stages": 4,
672 - }
673 - if M <= E:
674 - config = {
675 - "BLOCK_SIZE_M": 64,
676 - "BLOCK_SIZE_N": 128,
677 - "BLOCK_SIZE_K": 128,
678 - "GROUP_SIZE_M": 1,
679 - "num_warps": 4,
680 - "num_stages": 4,
681 - }
682 - else:
683 - # Block-wise quant: BLOCK_SIZE_N must be divisible by block_shape[0]
684 - # BLOCK_SIZE_K must be divisible by block_shape[1]
685 - config = {
686 - "BLOCK_SIZE_M": 64,
687 - "BLOCK_SIZE_N": block_shape[0],
688 - "BLOCK_SIZE_K": block_shape[1],
689 - "GROUP_SIZE_M": 32,
690 - "num_warps": 4,
691 - "num_stages": 3,
692 - }
663 + if dtype == "fp8_w8a8" and block_shape is not None:
664 + # Block-wise quant: BLOCK_SIZE_N must be divisible by block_shape[0]
665 + # BLOCK_SIZE_K must be divisible by block_shape[1]
666 + config = {
667 + "BLOCK_SIZE_M": 64,
668 + "BLOCK_SIZE_N": block_shape[0],
669 + "BLOCK_SIZE_K": block_shape[1],
670 + "GROUP_SIZE_M": 32,
671 + "num_warps": 4,
672 + "num_stages": 3,
673 + }
674 else:
675 config = {
676 "BLOCK_SIZE_M": 64,