| 1 | # MLA mixed batch benchmark (prefill + decode) |
| 2 | # Tests chunked prefill performance |
| 3 | |
| 4 | model: |
| 5 | name: "deepseek-v3" |
| 6 | num_layers: 60 |
| 7 | num_q_heads: 128 |
| 8 | num_kv_heads: 1 |
| 9 | head_dim: 576 |
| 10 | kv_lora_rank: 512 |
| 11 | qk_nope_head_dim: 128 |
| 12 | qk_rope_head_dim: 64 |
| 13 | v_head_dim: 128 |
| 14 | block_size: 128 |
| 15 | |
| 16 | batch_specs: |
| 17 | # Small prefill + decode |
| 18 | - "1q1k_8q1s1k" # 1 prefill + 8 decode |
| 19 | - "2q2k_16q1s1k" # 2 prefill + 16 decode |
| 20 | - "4q1k_32q1s2k" # 4 prefill + 32 decode |
| 21 | |
| 22 | # Medium prefill + decode |
| 23 | - "2q4k_32q1s2k" # 2 medium prefill + 32 decode |
| 24 | - "4q4k_64q1s2k" # 4 medium prefill + 64 decode |
| 25 | - "8q2k_64q1s4k" # 8 prefill + 64 decode |
| 26 | |
| 27 | # Large prefill + decode (chunked prefill stress test) |
| 28 | - "2q8k_32q1s1k" # 2 large prefill + 32 decode |
| 29 | - "1q16k_16q1s2k" # 1 very large prefill + 16 decode |
| 30 | - "2q16k_32q1s4k" # 2 very large prefill + 32 decode |
| 31 | |
| 32 | # Context extension + decode |
| 33 | - "2q1ks2k_16q1s1k" # 2 extend + 16 decode |
| 34 | - "4q2ks4k_32q1s2k" # 4 extend + 32 decode |
| 35 | - "2q1ks8k_32q1s2k" # 2 large extend + 32 decode |
| 36 | |
| 37 | # Explicitly chunked prefill |
| 38 | - "q8k" # 8k prefill with chunking hint |
| 39 | - "q16k" # 16k prefill with chunking hint |
| 40 | - "2q8k_32q1s2k" # 2 chunked prefill + 32 decode |
| 41 | |
| 42 | # High decode ratio (realistic serving) |
| 43 | - "1q2k_63q1s1k" # 1 prefill + 63 decode |
| 44 | - "2q2k_62q1s2k" # 2 prefill + 62 decode |
| 45 | - "4q4k_60q1s4k" # 4 prefill + 60 decode |
| 46 | |
| 47 | backends: |
| 48 | - CUTLASS_MLA |
| 49 | - FLASHINFER_MLA |
| 50 | - FLASH_ATTN_MLA # Hopper only |
| 51 | - FLASHMLA # Hopper only |
| 52 | |
| 53 | device: "cuda:0" |
| 54 | profile_memory: true |
| 55 | |
| 56 | # Analyze chunked prefill workspace size impact |
| 57 | chunked_prefill: |
| 58 | test_workspace_sizes: [4096, 8192, 16384, 32768, 65536] |