| 1 | # Speculative decoding benchmark configuration |
| 2 | # Tests reorder_batch_threshold optimization |
| 3 | |
| 4 | model: |
| 5 | name: "deepseek-v3" |
| 6 | num_layers: 60 |
| 7 | num_q_heads: 128 |
| 8 | num_kv_heads: 1 |
| 9 | head_dim: 576 |
| 10 | kv_lora_rank: 512 |
| 11 | qk_nope_head_dim: 128 |
| 12 | qk_rope_head_dim: 64 |
| 13 | v_head_dim: 128 |
| 14 | |
| 15 | batch_specs: |
| 16 | # Pure speculative decode (K-token verification) |
| 17 | - "q2s1k" # 2-token spec, 1k KV |
| 18 | - "q4s1k" # 4-token spec, 1k KV |
| 19 | - "q8s1k" # 8-token spec, 1k KV |
| 20 | - "q16s1k" # 16-token spec, 1k KV |
| 21 | |
| 22 | # Speculative with different context lengths |
| 23 | - "q4s2k" # 4-token spec, 2k KV |
| 24 | - "q4s4k" # 4-token spec, 4k KV |
| 25 | - "q8s2k" # 8-token spec, 2k KV |
| 26 | - "q8s4k" # 8-token spec, 4k KV |
| 27 | |
| 28 | # Mixed: speculative + regular decode |
| 29 | - "32q4s1k" # 32 spec requests |
| 30 | - "16q4s1k_16q1s1k" # 16 spec + 16 regular |
| 31 | - "8q8s2k_24q1s2k" # 8 spec (8-tok) + 24 regular |
| 32 | |
| 33 | # Mixed: speculative + prefill + decode |
| 34 | - "2q1k_16q4s1k_16q1s1k" # 2 prefill + 16 spec + 16 decode |
| 35 | - "4q2k_32q4s2k_32q1s2k" # 4 prefill + 32 spec + 32 decode |
| 36 | |
| 37 | # Large batches with speculation |
| 38 | - "64q4s1k" # 64 spec requests |
| 39 | - "32q8s2k" # 32 spec (8-token) |
| 40 | - "16q16s4k" # 16 spec (16-token) |
| 41 | |
| 42 | # Backends that support query length > 1 |
| 43 | backends: |
| 44 | - FLASH_ATTN_MLA # reorder_batch_threshold = 512 |
| 45 | - FLASHMLA # reorder_batch_threshold = 1 (tunable) |
| 46 | |
| 47 | # FlashInfer-MLA also supports uniform spec-as-decode but with different mechanism |
| 48 | # - FLASHINFER_MLA |
| 49 | |
| 50 | # Benchmark settings |
| 51 | device: "cuda:0" |
| 52 | profile_memory: false |
| 53 | |
| 54 | # Test these threshold values for optimization |
| 55 | parameter_sweep: |
| 56 | param_name: "reorder_batch_threshold" |
| 57 | values: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] |
| 58 | include_auto: false |
| 59 | label_format: "{backend}_threshold_{value}" |