A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Fix some speculative decode tests with tl.dot (#17371)
Signed-off-by: Huy Do <huydhn@gmail.com>
Huy Do committed
Apr 29, 2025 at 19:41 UTC
88fcf00ddaa99e9eb1da58c4d46dc5bf59bbf42d
1 file changed
+3
-6
tests/spec_decode/e2e/test_multistep_correctness.py
+3
-6
@@ -456,7 +456,7 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
456
@pytest.mark.parametrize(
457
"common_llm_kwargs",
458
[{
459
- "block_size": 8,
459
+ "block_size": 16,
460
# 2 for small prompt, 256//8 for generated.
461
"num_gpu_blocks_override": 2 + 256 // 8,
462
"max_model_len": (2 + 256 // 8) * 8,
@@ -526,11 +526,8 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
526
@pytest.mark.parametrize(
527
"per_test_common_llm_kwargs",
528
[
529
- # As of this writing, vLLM only compiles with these 3 block sizes by
530
- # default.
531
- {
532
- "block_size": 8,
533
- },
529
+ # https://github.com/triton-lang/triton/issues/2266 tl.dot
530
+ # doesn't support embedding < 16
531
{
532
"block_size": 16,
533
},