A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix][Mamba] Fix Multistep on Mamba-like models (#10705)
Signed-off-by: mzusman <mor.zusmann@gmail.com>
Mor Zusman committed
Nov 27, 2024 at 21:02 UTC
197b4484a3fba4a98921f903d6242677f97c63db
4 files changed
+84
-4
tests/models/decoder_only/language/test_jamba.py
+38
@@ -275,6 +275,44 @@ def test_state_cleanup(
275
"could be related to finished_requests_ids")
276
277
278
+@pytest.mark.parametrize("model", MODELS)
279
+@pytest.mark.parametrize("dtype", ["float"])
280
+def test_multistep(
281
+ vllm_runner,
282
+ model: str,
283
+ dtype: str,
284
+ example_prompts,
285
+) -> None:
286
+ # This test is verifying that multistep works correctly
287
+ #on mamba-like models
288
+ with vllm_runner(model, num_scheduler_steps=8,
289
+ max_num_seqs=2) as vllm_model:
290
+ vllm_model.generate_greedy([example_prompts[0]] * 10, 1)
291
+
292
+
293
+@pytest.mark.parametrize("model", MODELS)
294
+@pytest.mark.parametrize("dtype", ["float"])
295
+@pytest.mark.parametrize("max_tokens", [64])
296
+def test_multistep_correctness(vllm_runner, model: str, dtype: str,
297
+ max_tokens: int, example_prompts) -> None:
298
+ with vllm_runner(model, num_scheduler_steps=8,
299
+ max_num_seqs=2) as vllm_model:
300
+ vllm_outputs_multistep = vllm_model.generate_greedy(
301
+ example_prompts, max_tokens)
302
+
303
+ with vllm_runner(model, num_scheduler_steps=1,
304
+ max_num_seqs=2) as vllm_model:
305
+ vllm_outputs_single_step = vllm_model.generate_greedy(
306
+ example_prompts, max_tokens)
307
+
308
+ check_outputs_equal(
309
+ outputs_0_lst=vllm_outputs_multistep,
310
+ outputs_1_lst=vllm_outputs_single_step,
311
+ name_0="vllm_outputs_multistep",
312
+ name_1="vllm_outputs_single_step",
313
+ )
314
+
315
+
316
@multi_gpu_test(num_gpus=2)
317
@pytest.mark.parametrize("model", MODELS)
318
@pytest.mark.parametrize("dtype", ["float"])
tests/models/decoder_only/language/test_mamba.py
+36
@@ -283,3 +283,39 @@ def test_state_cleanup(
283
except ValueError:
284
pytest.fail("Mamba inner state wasn't cleaned up between states, "
285
"could be related to finished_requests_ids")
286
+
287
+
288
+@pytest.mark.parametrize("model", MODELS)
289
+@pytest.mark.parametrize("dtype", ["float"])
290
+def test_multistep(
291
+ vllm_runner,
292
+ model: str,
293
+ dtype: str,
294
+ example_prompts,
295
+) -> None:
296
+ with vllm_runner(model, num_scheduler_steps=8,
297
+ max_num_seqs=2) as vllm_model:
298
+ vllm_model.generate_greedy([example_prompts[0]] * 10, 1)
299
+
300
+
301
+@pytest.mark.parametrize("model", MODELS)
302
+@pytest.mark.parametrize("dtype", ["float"])
303
+@pytest.mark.parametrize("max_tokens", [64])
304
+def test_multistep_correctness(vllm_runner, model: str, dtype: str,
305
+ max_tokens: int, example_prompts) -> None:
306
+ with vllm_runner(model, num_scheduler_steps=8,
307
+ max_num_seqs=2) as vllm_model:
308
+ vllm_outputs_multistep = vllm_model.generate_greedy(
309
+ example_prompts, max_tokens)
310
+
311
+ with vllm_runner(model, num_scheduler_steps=1,
312
+ max_num_seqs=2) as vllm_model:
313
+ vllm_outputs_single_step = vllm_model.generate_greedy(
314
+ example_prompts, max_tokens)
315
+
316
+ check_outputs_equal(
317
+ outputs_0_lst=vllm_outputs_multistep,
318
+ outputs_1_lst=vllm_outputs_single_step,
319
+ name_0="vllm_outputs_multistep",
320
+ name_1="vllm_outputs_single_step",
321
+ )
vllm/engine/async_llm_engine.py
+5
-2
@@ -300,6 +300,9 @@ class _AsyncLLMEngine(LLMEngine):
300
ctx.seq_group_metadata_list = seq_group_metadata_list
301
ctx.scheduler_outputs = scheduler_outputs
302
303
+ finished_requests_ids = self.scheduler[
304
+ virtual_engine].get_and_reset_finished_requests_ids()
305
+
306
# Maybe switch from async mode to sync mode
307
if not allow_async_output_proc and len(ctx.output_queue) > 0:
308
self._process_model_outputs(ctx=ctx)
@@ -311,13 +314,13 @@ class _AsyncLLMEngine(LLMEngine):
314
self._cache_scheduler_outputs_for_multi_step(
315
virtual_engine, seq_group_metadata_list, scheduler_outputs,
316
allow_async_output_proc)
317
+ else:
318
+ finished_requests_ids = list()
319
320
assert seq_group_metadata_list is not None
321
assert scheduler_outputs is not None
322
323
if not scheduler_outputs.is_empty():
319
- finished_requests_ids = self.scheduler[
320
- virtual_engine].get_and_reset_finished_requests_ids()
324
325
# Check if we have a cached last_output from the previous iteration.
326
# For supporting PP this is probably the best way to pass the
vllm/engine/llm_engine.py
+5
-2
@@ -1398,6 +1398,9 @@ class LLMEngine:
1398
ctx.seq_group_metadata_list = seq_group_metadata_list
1399
ctx.scheduler_outputs = scheduler_outputs
1400
1401
+ finished_requests_ids = self.scheduler[
1402
+ virtual_engine].get_and_reset_finished_requests_ids()
1403
+
1404
# Maybe switch from async mode to sync mode
1405
if not allow_async_output_proc and len(ctx.output_queue) > 0:
1406
self._process_model_outputs(ctx=ctx)
@@ -1409,13 +1412,13 @@ class LLMEngine:
1412
self._cache_scheduler_outputs_for_multi_step(
1413
virtual_engine, seq_group_metadata_list, scheduler_outputs,
1414
allow_async_output_proc)
1415
+ else:
1416
+ finished_requests_ids = list()
1417
1418
assert seq_group_metadata_list is not None
1419
assert scheduler_outputs is not None
1420
1421
if not scheduler_outputs.is_empty():
1417
- finished_requests_ids = self.scheduler[
1418
- virtual_engine].get_and_reset_finished_requests_ids()
1422
1423
# Check if we have a cached last_output from the previous iteration.
1424
# For supporting PP this is probably the best way to pass the