A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Revert MoE Triton Config Default (#12629)
SUMMARY: * previous PR for pulling in block configs also changed defaults (https://github.com/vllm-project/vllm/pull/11589/files) for FP8 * this broke L4 MoE since there was not enough SHM for the default configuration * this reverts the non-block example to the default Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw committed
Jan 31, 2025 at 18:28 UTC
145c2ff648ad0a300f880ac38811d0d8a2eb3e79
1 file changed
+11
-30
vllm/model_executor/layers/fused_moe/fused_moe.py
+11
-30
@@ -660,36 +660,17 @@ def get_default_config(
660
is_marlin: bool,
661
block_shape: Optional[List[int]] = None,
662
) -> Dict[str, int]:
663
- if dtype == "fp8_w8a8":
664
- if block_shape is None:
665
- config = {
666
- "BLOCK_SIZE_M": 128,
667
- "BLOCK_SIZE_N": 256,
668
- "BLOCK_SIZE_K": 128,
669
- "GROUP_SIZE_M": 32,
670
- "num_warps": 8,
671
- "num_stages": 4,
672
- }
673
- if M <= E:
674
- config = {
675
- "BLOCK_SIZE_M": 64,
676
- "BLOCK_SIZE_N": 128,
677
- "BLOCK_SIZE_K": 128,
678
- "GROUP_SIZE_M": 1,
679
- "num_warps": 4,
680
- "num_stages": 4,
681
- }
682
- else:
683
- # Block-wise quant: BLOCK_SIZE_N must be divisible by block_shape[0]
684
- # BLOCK_SIZE_K must be divisible by block_shape[1]
685
- config = {
686
- "BLOCK_SIZE_M": 64,
687
- "BLOCK_SIZE_N": block_shape[0],
688
- "BLOCK_SIZE_K": block_shape[1],
689
- "GROUP_SIZE_M": 32,
690
- "num_warps": 4,
691
- "num_stages": 3,
692
- }
663
+ if dtype == "fp8_w8a8" and block_shape is not None:
664
+ # Block-wise quant: BLOCK_SIZE_N must be divisible by block_shape[0]
665
+ # BLOCK_SIZE_K must be divisible by block_shape[1]
666
+ config = {
667
+ "BLOCK_SIZE_M": 64,
668
+ "BLOCK_SIZE_N": block_shape[0],
669
+ "BLOCK_SIZE_K": block_shape[1],
670
+ "GROUP_SIZE_M": 32,
671
+ "num_warps": 4,
672
+ "num_stages": 3,
673
+ }
674
else:
675
config = {
676
"BLOCK_SIZE_M": 64,