A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[ROCm][MoE] mi300 mixtral8x7B perf for specific BS (#13577)
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Divakar Verma committed
Feb 19, 2025 at 22:01 UTC
0d243f2a54fbd1c56da8a571f0899c30b6aba5d9
3 files changed
+5
-5
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+2
-2
@@ -45,8 +45,8 @@
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
48
- "BLOCK_SIZE_N": 16,
49
- "BLOCK_SIZE_K": 256,
48
+ "BLOCK_SIZE_N": 64,
49
+ "BLOCK_SIZE_K": 64,
50
"GROUP_SIZE_M": 1,
51
"num_warps": 2,
52
"num_stages": 2,
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+2
-2
@@ -45,8 +45,8 @@
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
48
- "BLOCK_SIZE_N": 32,
49
- "BLOCK_SIZE_K": 256,
48
+ "BLOCK_SIZE_N": 64,
49
+ "BLOCK_SIZE_K": 128,
50
"GROUP_SIZE_M": 1,
51
"num_warps": 2,
52
"num_stages": 2,
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+1
-1
@@ -128,7 +128,7 @@
128
"num_warps": 8,
129
"num_stages": 2,
130
"waves_per_eu": 0,
131
- "matrix_instr_nonkdim": 32,
131
+ "matrix_instr_nonkdim": 16,
132
"kpack": 2
133
},
134
"512": {