A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[ROCm][AMD][Bugfix] adding a missing triton autotune config (#4845)
Hongxia Yang committed
May 16, 2024 at 13:46 UTC
b5853f99639afd82cb18f131dce7f8c41eda74bd
1 file changed
+10
vllm/attention/ops/triton_flash_attention.py
+10
@@ -239,6 +239,16 @@ def _attn_fwd_inner(
239
num_stages=1,
240
num_warps=8,
241
),
242
+ triton.Config(
243
+ {
244
+ "BLOCK_M": 128,
245
+ "BLOCK_N": 64,
246
+ "waves_per_eu": 1,
247
+ "PRE_LOAD_V": False,
248
+ },
249
+ num_stages=1,
250
+ num_warps=4,
251
+ ),
252
triton.Config(
253
{
254
"BLOCK_M": 128,