A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[BugFix][Model] Jamba - Handle aborted requests, Add tests and fix cleanup bug (#6425)
Co-authored-by: Mor Zusman <morz@ai21.com>
Mor Zusman committed
Jul 16, 2024 at 04:32 UTC
9ad32dacd9c4697ca06e841bd1bc97b58d2c7b3d
5 files changed
+176
-24
tests/models/test_jamba.py
+83
@@ -1,5 +1,6 @@
1
import pytest
2
3
+from tests.models.utils import check_outputs_equal
4
from vllm.worker.model_runner import _get_graph_batch_size
5
6
MODELS = ["ai21labs/Jamba-tiny-random"]
@@ -34,6 +35,34 @@ def test_models(
35
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
36
37
38
+@pytest.mark.parametrize("model", MODELS)
39
+@pytest.mark.parametrize("dtype", ["float"])
40
+@pytest.mark.parametrize("max_tokens", [20])
41
+def test_batching(
42
+ vllm_runner,
43
+ example_prompts,
44
+ model: str,
45
+ dtype: str,
46
+ max_tokens: int,
47
+) -> None:
48
+ # To pass the small model tests, we need full precision.
49
+ for_loop_outputs = []
50
+ with vllm_runner(model, dtype=dtype) as vllm_model:
51
+ for prompt in example_prompts:
52
+ for_loop_outputs.append(
53
+ vllm_model.generate_greedy([prompt], max_tokens)[0])
54
+
55
+ batched_outputs = vllm_model.generate_greedy(example_prompts,
56
+ max_tokens)
57
+
58
+ check_outputs_equal(
59
+ outputs_0_lst=for_loop_outputs,
60
+ outputs_1_lst=batched_outputs,
61
+ name_0="for_loop_vllm",
62
+ name_1="batched_vllm",
63
+ )
64
+
65
+
66
@pytest.mark.parametrize("model", MODELS)
67
@pytest.mark.parametrize("dtype", ["bfloat16"])
68
@pytest.mark.parametrize("max_tokens", [20])
@@ -60,6 +89,60 @@ def test_mamba_cache_cg_padding(
89
"Could be related to mamba cache not padded correctly")
90
91
92
+@pytest.mark.parametrize("model", MODELS)
93
+@pytest.mark.parametrize("dtype", ["float"])
94
+@pytest.mark.parametrize("max_tokens", [20])
95
+def test_models_preemption_recompute(
96
+ hf_runner,
97
+ vllm_runner,
98
+ example_prompts,
99
+ model: str,
100
+ dtype: str,
101
+ max_tokens: int,
102
+) -> None:
103
+ # Tests that outputs are identical with and w/o preemtions (recompute)
104
+ assert dtype == "float"
105
+
106
+ with vllm_runner(model, dtype=dtype) as vllm_model:
107
+ vllm_model.model.llm_engine.scheduler[
108
+ 0].ENABLE_ARTIFICIAL_PREEMPT = True
109
+ preempt_vllm_outputs = vllm_model.generate_greedy(
110
+ example_prompts, max_tokens)
111
+
112
+ vllm_model.model.llm_engine.scheduler[
113
+ 0].ENABLE_ARTIFICIAL_PREEMPT = False
114
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
115
+
116
+ check_outputs_equal(
117
+ outputs_0_lst=preempt_vllm_outputs,
118
+ outputs_1_lst=vllm_outputs,
119
+ name_0="vllm_preepmtions",
120
+ name_1="vllm",
121
+ )
122
+
123
+
124
+@pytest.mark.parametrize("model", MODELS)
125
+@pytest.mark.parametrize("dtype", ["float"])
126
+def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
127
+ vllm_runner,
128
+ model: str,
129
+ dtype: str,
130
+ example_prompts,
131
+) -> None:
132
+ # This test is for verifying that the Jamba inner state management doesn't
133
+ # collapse in case where the number of incoming requests and
134
+ # finished_requests_ids is larger than the maximum mamba block capacity.
135
+ # This could generally happen due to the fact that Jamba does support
136
+ # statelessness mechanism where it can cleanup new incoming requests in
137
+ # a single step.
138
+ try:
139
+ with vllm_runner(model, dtype=dtype, max_num_seqs=10) as vllm_model:
140
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
141
+ except ValueError:
142
+ pytest.fail("Jamba inner state wasn't cleaned up properly between"
143
+ "steps finished requests registered unnecessarily ")
144
+
145
+
146
@pytest.mark.parametrize("model", MODELS)
147
@pytest.mark.parametrize("dtype", ["float"])
148
def test_state_cleanup(
vllm/core/scheduler.py
+1
@@ -374,6 +374,7 @@ class Scheduler:
374
for aborted_group in aborted_groups:
375
# Remove the sequence group from the state queue.
376
state_queue.remove(aborted_group)
377
+ self._finished_requests_ids.append(aborted_group.request_id)
378
for seq in aborted_group.get_seqs():
379
if seq.is_finished():
380
continue
vllm/model_executor/model_loader/loader.py
+20
-12
@@ -32,7 +32,8 @@ from vllm.model_executor.model_loader.weight_utils import (
32
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
33
get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34
pt_weights_iterator, safetensors_weights_iterator)
35
-from vllm.model_executor.models.interfaces import (supports_lora,
35
+from vllm.model_executor.models.interfaces import (has_inner_state,
36
+ supports_lora,
37
supports_vision)
38
from vllm.model_executor.utils import set_weight_attrs
39
from vllm.platforms import current_platform
@@ -66,10 +67,10 @@ def _get_quantization_config(
67
68
69
def _get_model_initialization_kwargs(
69
- model_class: Type[nn.Module],
70
- lora_config: Optional[LoRAConfig],
71
- multimodal_config: Optional[MultiModalConfig],
72
-) -> Dict[str, Any]:
70
+ model_class: Type[nn.Module],
71
+ lora_config: Optional[LoRAConfig],
72
+ multimodal_config: Optional[MultiModalConfig],
73
+ scheduler_config: Optional[SchedulerConfig] = None) -> Dict[str, Any]:
74
"""Get extra kwargs for model initialization."""
75
extra_kwargs: Dict[str, Any] = {}
76
@@ -90,13 +91,19 @@ def _get_model_initialization_kwargs(
91
92
extra_kwargs["multimodal_config"] = multimodal_config
93
94
+ if has_inner_state(model_class) and scheduler_config:
95
+ extra_kwargs["scheduler_config"] = scheduler_config
96
+
97
return extra_kwargs
98
99
96
-def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
97
- lora_config: Optional[LoRAConfig],
98
- multimodal_config: Optional[MultiModalConfig],
99
- cache_config: CacheConfig) -> nn.Module:
100
+def _initialize_model(
101
+ model_config: ModelConfig,
102
+ load_config: LoadConfig,
103
+ lora_config: Optional[LoRAConfig],
104
+ multimodal_config: Optional[MultiModalConfig],
105
+ cache_config: CacheConfig,
106
+ scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
107
"""Initialize a model with the given configurations."""
108
model_class = get_model_architecture(model_config)[0]
109
quant_config = _get_quantization_config(model_config, load_config)
@@ -105,7 +112,8 @@ def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
112
cache_config=cache_config,
113
quant_config=quant_config,
114
**_get_model_initialization_kwargs(
108
- model_class, lora_config, multimodal_config))
115
+ model_class, lora_config, multimodal_config,
116
+ scheduler_config))
117
118
119
class BaseModelLoader(ABC):
@@ -266,7 +274,7 @@ class DefaultModelLoader(BaseModelLoader):
274
with torch.device(device_config.device):
275
model = _initialize_model(model_config, self.load_config,
276
lora_config, multimodal_config,
269
- cache_config)
277
+ cache_config, scheduler_config)
278
model.load_weights(
279
self._get_weights_iterator(model_config.model,
280
model_config.revision,
@@ -302,7 +310,7 @@ class DummyModelLoader(BaseModelLoader):
310
with torch.device(device_config.device):
311
model = _initialize_model(model_config, self.load_config,
312
lora_config, multimodal_config,
305
- cache_config)
313
+ cache_config, scheduler_config)
314
# NOTE(woosuk): For accurate performance evaluation, we assign
315
# random values to the weights.
316
initialize_dummy_weights(model)
vllm/model_executor/models/interfaces.py
+47
-1
@@ -3,7 +3,7 @@ from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
3
4
from typing_extensions import TypeGuard
5
6
-from vllm.config import LoRAConfig, MultiModalConfig
6
+from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
7
from vllm.logger import init_logger
8
9
logger = init_logger(__name__)
@@ -142,3 +142,49 @@ def _supports_lora(
142
return isinstance(model, _SupportsLoRAType)
143
144
return isinstance(model, SupportsLoRA)
145
+
146
+
147
+@runtime_checkable
148
+class HasInnerState(Protocol):
149
+ """The interface required for all models that has inner state."""
150
+
151
+ has_inner_state: ClassVar[Literal[True]] = True
152
+ """
153
+ A flag that indicates this model has inner state.
154
+ Models that has inner state usually need access to the scheduler_config
155
+ for max_num_seqs ,etc... (Currently only used by Jamba)
156
+ """
157
+
158
+ def __init__(self,
159
+ *,
160
+ scheduler_config: Optional[SchedulerConfig] = None) -> None:
161
+ ...
162
+
163
+
164
+@runtime_checkable
165
+class _HasInnerStateType(Protocol):
166
+ has_inner_state: ClassVar[Literal[True]]
167
+
168
+ def __init__(self,
169
+ *,
170
+ scheduler_config: Optional[SchedulerConfig] = None) -> None:
171
+ ...
172
+
173
+
174
+@overload
175
+def has_inner_state(model: object) -> TypeGuard[HasInnerState]:
176
+ ...
177
+
178
+
179
+@overload
180
+def has_inner_state(model: Type[object]) -> TypeGuard[Type[HasInnerState]]:
181
+ ...
182
+
183
+
184
+def has_inner_state(
185
+ model: Union[Type[object], object]
186
+) -> Union[TypeGuard[Type[HasInnerState]], TypeGuard[HasInnerState]]:
187
+ if isinstance(model, type):
188
+ return isinstance(model, _HasInnerStateType)
189
+
190
+ return isinstance(model, HasInnerState)
vllm/model_executor/models/jamba.py
+25
-11
@@ -13,7 +13,7 @@ from transformers import JambaConfig
13
14
from vllm.attention.backends.abstract import AttentionMetadata
15
from vllm.attention.layer import Attention
16
-from vllm.config import CacheConfig, LoRAConfig
16
+from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
17
from vllm.distributed import (get_tensor_model_parallel_rank,
18
get_tensor_model_parallel_world_size,
19
tensor_model_parallel_all_reduce)
@@ -32,10 +32,12 @@ from vllm.model_executor.layers.sampler import Sampler
32
from vllm.model_executor.layers.vocab_parallel_embedding import (
33
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
34
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
+from vllm.model_executor.models.interfaces import HasInnerState
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.model_executor.utils import set_weight_attrs
38
from vllm.sequence import IntermediateTensors, SamplerOutput
38
-from vllm.worker.model_runner import _BATCH_SIZES_TO_CAPTURE
39
+from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
40
+ _get_graph_batch_size)
41
42
KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -612,7 +614,7 @@ class JambaModel(nn.Module):
614
return hidden_states
615
616
615
-class JambaForCausalLM(nn.Module):
617
+class JambaForCausalLM(nn.Module, HasInnerState):
618
packed_modules_mapping = {
619
"qkv_proj": [
620
"q_proj",
@@ -640,9 +642,11 @@ class JambaForCausalLM(nn.Module):
642
cache_config: Optional[CacheConfig] = None,
643
quant_config: Optional[QuantizationConfig] = None,
644
lora_config: Optional[LoRAConfig] = None,
645
+ scheduler_config: Optional[SchedulerConfig] = None,
646
) -> None:
647
super().__init__()
648
self.config = config
649
+ self.scheduler_config = scheduler_config
650
self.model = JambaModel(config,
651
cache_config=cache_config,
652
quant_config=quant_config,
@@ -689,6 +693,8 @@ class JambaForCausalLM(nn.Module):
693
for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
694
695
request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
696
+ finished_requests_ids = kwargs["finished_requests_ids"]
697
+ self._release_mamba_cache(finished_requests_ids)
698
batch_size = input_ids.shape[0]
699
if attn_metadata.prefill_metadata:
700
batch_size = len(request_ids_to_seq_ids)
@@ -696,9 +702,8 @@ class JambaForCausalLM(nn.Module):
702
current_seqlen_agnostic_cache,
703
indices,
704
) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
699
- batch_size)
700
- finished_requests_ids = kwargs["finished_requests_ids"]
701
- self._release_mamba_cache(finished_requests_ids)
705
+ batch_size,
706
+ finished_requests_ids)
707
else:
708
# CUDA graph capturing runs
709
current_seqlen_agnostic_cache, indices = (
@@ -760,10 +765,15 @@ class JambaForCausalLM(nn.Module):
765
return indices_for_current_run
766
767
def _prepare_current_run_mamba_cache(
763
- self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int
768
+ self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int,
769
+ finished_requests_ids: List[str]
770
) -> Tuple[Tuple[torch.Tensor, torch.Tensor], List[int]]:
771
indices_for_current_run = []
772
for request_id, seqs_id in request_ids_to_seq_ids.items():
773
+ if request_id in finished_requests_ids:
774
+ # Do not allocate cache for requests that run
775
+ # and finish right after
776
+ continue
777
indices_for_current_run += self._assign_seq_id_to_mamba_cache(
778
request_id, seqs_id)
779
## Pad the batch in case of running batch that was not captured via CG
@@ -787,16 +797,17 @@ class JambaForCausalLM(nn.Module):
797
assert all(
798
key in kwargs
799
for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
800
+ finished_requests_ids = kwargs["finished_requests_ids"]
801
+ self._release_mamba_cache(finished_requests_ids)
802
request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
803
cg_batch_size = input_buffers['input_ids'].shape[0]
804
(
805
current_mamba_cache,
806
indices,
807
) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796
- cg_batch_size)
808
+ cg_batch_size,
809
+ finished_requests_ids)
810
self.current_indices = indices
798
- finished_requests_ids = kwargs["finished_requests_ids"]
799
- self._release_mamba_cache(finished_requests_ids)
811
812
for input_buffer, current_cache_buffer in zip(
813
input_buffers["seqlen_agnostic_capture_inputs"],
@@ -860,9 +871,12 @@ class JambaForCausalLM(nn.Module):
871
layers_type = self.config.layers_block_type
872
mamba_layers = sum(
873
[layer_type == "mamba" for layer_type in layers_type])
863
- max_batch_size = _BATCH_SIZES_TO_CAPTURE[-1] + 10
874
+ max_batch_size = (_get_graph_batch_size(
875
+ self.scheduler_config.max_num_seqs) if self.scheduler_config else
876
+ max(_BATCH_SIZES_TO_CAPTURE)) + 10
877
conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
878
assert conv_state_shape is not None and temporal_state_shape is not None
879
+
880
for buffername in ["mamba_cache", "mamba_gc_cache_buffer"]:
881
buffer = (torch.empty(size=(mamba_layers, max_batch_size) +
882
conv_state_shape,