A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Mamba cache Cuda Graph padding (#6214)
tomeras91 committed
Jul 8, 2024 at 21:25 UTC
ddc369fba147046f5044aaddbb867b5333f7068c
2 files changed
+30
-2
tests/models/test_jamba.py
+28
@@ -1,5 +1,7 @@
1
import pytest
2
3
+from vllm.worker.model_runner import _get_graph_batch_size
4
+
5
MODELS = ["ai21labs/Jamba-tiny-random"]
6
7
@@ -32,6 +34,32 @@ def test_models(
34
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
35
36
37
+@pytest.mark.parametrize("model", MODELS)
38
+@pytest.mark.parametrize("dtype", ["bfloat16"])
39
+@pytest.mark.parametrize("max_tokens", [20])
40
+def test_mamba_cache_cg_padding(
41
+ vllm_runner,
42
+ example_prompts,
43
+ model: str,
44
+ dtype: str,
45
+ max_tokens: int,
46
+) -> None:
47
+ # This test is for verifying that mamba cache is padded to CG captured
48
+ # batch size. If it's not, a torch RuntimeError will be raised because
49
+ # tensor dimensions aren't compatible
50
+ while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
51
+ example_prompts.append(example_prompts[0])
52
+
53
+ try:
54
+ with vllm_runner(model, dtype=dtype) as vllm_model:
55
+ vllm_model.generate_greedy(example_prompts, max_tokens)
56
+ except RuntimeError:
57
+ pytest.fail(
58
+ "Couldn't run batch size which is not equal to a Cuda Graph "
59
+ "captured batch size. "
60
+ "Could be related to mamba cache not padded correctly")
61
+
62
+
63
@pytest.mark.parametrize("model", MODELS)
64
@pytest.mark.parametrize("dtype", ["float"])
65
def test_state_cleanup(
vllm/model_executor/models/jamba.py
+2
-2
@@ -788,12 +788,12 @@ class JambaForCausalLM(nn.Module):
788
key in kwargs
789
for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
790
request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
791
- batch_size = len(request_ids_to_seq_ids)
791
+ cg_batch_size = input_buffers['input_ids'].shape[0]
792
(
793
current_mamba_cache,
794
indices,
795
) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796
- batch_size)
796
+ cg_batch_size)
797
self.current_indices = indices
798
finished_requests_ids = kwargs["finished_requests_ids"]
799
self._release_mamba_cache(finished_requests_ids)