| 1 | # Standard attention backend benchmark configuration |
| 2 | |
| 3 | model: |
| 4 | num_layers: 32 |
| 5 | num_q_heads: 32 |
| 6 | num_kv_heads: 8 # GQA with 4:1 ratio |
| 7 | head_dim: 128 |
| 8 | block_size: 16 |
| 9 | |
| 10 | batch_specs: |
| 11 | # Pure prefill |
| 12 | - "q512" # Small prefill (512 tokens) |
| 13 | - "q2k" # Medium prefill (2048 tokens) |
| 14 | - "q4k" # Large prefill (4096 tokens) |
| 15 | - "q8k" # Very large prefill (8192 tokens) |
| 16 | |
| 17 | # Pure decode |
| 18 | - "8q1s1k" # 8 requests, 1k KV cache each |
| 19 | - "16q1s2k" # 16 requests, 2k KV cache each |
| 20 | - "32q1s1k" # 32 requests, 1k KV cache each |
| 21 | - "64q1s4k" # 64 requests, 4k KV cache each |
| 22 | |
| 23 | # Mixed prefill/decode |
| 24 | - "2q2k_8q1s1k" # 2 prefill + 8 decode |
| 25 | - "4q1k_16q1s2k" # 4 prefill + 16 decode |
| 26 | - "2q4k_32q1s1k" # 2 large prefill + 32 decode |
| 27 | |
| 28 | # Speculative decode (q <= 8) |
| 29 | - "16q2s1k" # 16 requests, 2 spec tokens, 1k KV cache |
| 30 | - "16q4s1k" # 16 requests, 4 spec tokens, 1k KV cache |
| 31 | - "16q8s1k" # 16 requests, 8 spec tokens, 1k KV cache |
| 32 | - "32q4s2k" # 32 requests, 4 spec tokens, 2k KV cache |
| 33 | - "8q8s4k" # 8 requests, 8 spec tokens, 4k KV cache |
| 34 | |
| 35 | # Context extension (chunked prefill) |
| 36 | - "q1ks2k" # 1k query, 2k sequence |
| 37 | - "2q1ks4k" # 2 requests: 1k query, 4k sequence |
| 38 | |
| 39 | # Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER |
| 40 | backends: |
| 41 | - FLASH_ATTN |
| 42 | - TRITON_ATTN |
| 43 | - FLASHINFER |
| 44 | |
| 45 | device: "cuda:0" |
| 46 | profile_memory: false |