@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f1fc7425a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][CI/Test] Do basic test for top-p & top-k sampling (#12469)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 27, 2025 at 09:40 UTC 3f1fc7425a7db4d9722941075e43bb2ebfb90613
1 file changed +20 -8
tests/v1/engine/test_engine_core.py
+20 -8
@@ -144,7 +144,7 @@ def test_engine_core(monkeypatch):
144 def test_engine_core_advanced_sampling(monkeypatch):
145 """
146 A basic end-to-end test to verify that the engine functions correctly
147 - when additional sampling parameters, such as min_tokens and
147 + when additional sampling parameters, such as top_p, min_tokens, and
148 presence_penalty, are set.
149 """
150 with monkeypatch.context() as m:
@@ -167,11 +167,23 @@ def test_engine_core_advanced_sampling(monkeypatch):
167 stop_token_ids=[1001, 1002],
168 )
169 engine_core.add_request(request)
170 - assert len(engine_core.scheduler.waiting) == 1
171 - assert len(engine_core.scheduler.running) == 0
172 - # Loop through until they are all done.
173 - while len(engine_core.step().outputs) > 0:
174 - pass
170
176 - assert len(engine_core.scheduler.waiting) == 0
177 - assert len(engine_core.scheduler.running) == 0
171 + def _check_engine_state():
172 + assert len(engine_core.scheduler.waiting) == 1
173 + assert len(engine_core.scheduler.running) == 0
174 + # Loop through until they are all done.
175 + while len(engine_core.step().outputs) > 0:
176 + pass
177 + assert len(engine_core.scheduler.waiting) == 0
178 + assert len(engine_core.scheduler.running) == 0
179 +
180 + _check_engine_state()
181 +
182 + # Second request.
183 + request2 = make_request()
184 + request2.sampling_params = SamplingParams(
185 + top_p=0.99,
186 + top_k=50,
187 + )
188 + engine_core.add_request(request2)
189 + _check_engine_state()