A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Fix test_memory_usage_no_spec (#17754)
Signed-off-by: Yong Hoon Shin <yhshin@meta.com>
Yong Hoon Shin committed
May 7, 2025 at 00:10 UTC
324a3119b0d738b6fd2e1e5b46ff144411491b4c
1 file changed
+6
-6
tests/spec_decode/test_memory_usage.py
+6
-6
@@ -42,12 +42,12 @@ we can ensure we go through the _no_spec codepath for most of our engine steps.
42
43
def test_memory_usage_no_spec():
44
previous_memory_allocated = None
45
- llm = vllm.LLM(
46
- model=MAIN_MODEL,
47
- speculative_model=SPEC_MODEL,
48
- num_speculative_tokens=3,
49
- speculative_disable_by_batch_size=SPEC_DISABLE_BATCH_SIZE,
50
- )
45
+ llm = vllm.LLM(model=MAIN_MODEL,
46
+ speculative_config={
47
+ "model": SPEC_MODEL,
48
+ "num_speculative_tokens": 3,
49
+ "disable_by_batch_size": SPEC_DISABLE_BATCH_SIZE,
50
+ })
51
52
batch_sequences = set()
53
engine = llm.llm_engine