@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ddc369fba1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Mamba cache Cuda Graph padding (#6214)

tomeras91 committed Jul 8, 2024 at 21:25 UTC ddc369fba147046f5044aaddbb867b5333f7068c
2 files changed +30 -2
tests/models/test_jamba.py
+28
@@ -1,5 +1,7 @@
1 import pytest
2
3 +from vllm.worker.model_runner import _get_graph_batch_size
4 +
5 MODELS = ["ai21labs/Jamba-tiny-random"]
6
7
@@ -32,6 +34,32 @@ def test_models(
34 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
35
36
37 +@pytest.mark.parametrize("model", MODELS)
38 +@pytest.mark.parametrize("dtype", ["bfloat16"])
39 +@pytest.mark.parametrize("max_tokens", [20])
40 +def test_mamba_cache_cg_padding(
41 + vllm_runner,
42 + example_prompts,
43 + model: str,
44 + dtype: str,
45 + max_tokens: int,
46 +) -> None:
47 + # This test is for verifying that mamba cache is padded to CG captured
48 + # batch size. If it's not, a torch RuntimeError will be raised because
49 + # tensor dimensions aren't compatible
50 + while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
51 + example_prompts.append(example_prompts[0])
52 +
53 + try:
54 + with vllm_runner(model, dtype=dtype) as vllm_model:
55 + vllm_model.generate_greedy(example_prompts, max_tokens)
56 + except RuntimeError:
57 + pytest.fail(
58 + "Couldn't run batch size which is not equal to a Cuda Graph "
59 + "captured batch size. "
60 + "Could be related to mamba cache not padded correctly")
61 +
62 +
63 @pytest.mark.parametrize("model", MODELS)
64 @pytest.mark.parametrize("dtype", ["float"])
65 def test_state_cleanup(
vllm/model_executor/models/jamba.py
+2 -2
@@ -788,12 +788,12 @@ class JambaForCausalLM(nn.Module):
788 key in kwargs
789 for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
790 request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
791 - batch_size = len(request_ids_to_seq_ids)
791 + cg_batch_size = input_buffers['input_ids'].shape[0]
792 (
793 current_mamba_cache,
794 indices,
795 ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796 - batch_size)
796 + cg_batch_size)
797 self.current_indices = indices
798 finished_requests_ids = kwargs["finished_requests_ids"]
799 self._release_mamba_cache(finished_requests_ids)