@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bce324487a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[CI][SpecDecode] Fix spec decode tests, use flash attention backend for spec decode CI tests. (#8975)

Lily Liu committed Sep 30, 2024 at 17:51 UTC bce324487a8e36140143ea37f4b27d273a0fd661
2 files changed +4 -3
.buildkite/test-pipeline.yaml
-2
@@ -207,8 +207,6 @@ steps:
207 - vllm/spec_decode
208 - tests/spec_decode
209 commands:
210 - # See https://github.com/vllm-project/vllm/issues/5152
211 - - export VLLM_ATTENTION_BACKEND=XFORMERS
210 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
211 - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
212
tests/spec_decode/test_multi_step_worker.py
+4 -1
@@ -673,7 +673,10 @@ def test_use_draft_model_runner_advance_step():
673 worker.model_runner._gpu_advance_step.side_effect = ValueError(
674 exception_secret)
675
676 - seq_group_metadata_list, _, _ = create_batch(batch_size, k)
676 + seq_group_metadata_list, _, _ = create_batch(batch_size,
677 + k,
678 + block_size=block_size,
679 + num_gpu_blocks=num_gpu_blocks)
680
681 # Fallback (should not call) when num_steps=1.
682 execute_model_req = ExecuteModelRequest(