@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ad32dacd9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[BugFix][Model] Jamba - Handle aborted requests, Add tests and fix cleanup bug (#6425)

Co-authored-by: Mor Zusman <morz@ai21.com>

Mor Zusman committed Jul 16, 2024 at 04:32 UTC 9ad32dacd9c4697ca06e841bd1bc97b58d2c7b3d
5 files changed +176 -24
tests/models/test_jamba.py
+83
@@ -1,5 +1,6 @@
1 import pytest
2
3 +from tests.models.utils import check_outputs_equal
4 from vllm.worker.model_runner import _get_graph_batch_size
5
6 MODELS = ["ai21labs/Jamba-tiny-random"]
@@ -34,6 +35,34 @@ def test_models(
35 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
36
37
38 +@pytest.mark.parametrize("model", MODELS)
39 +@pytest.mark.parametrize("dtype", ["float"])
40 +@pytest.mark.parametrize("max_tokens", [20])
41 +def test_batching(
42 + vllm_runner,
43 + example_prompts,
44 + model: str,
45 + dtype: str,
46 + max_tokens: int,
47 +) -> None:
48 + # To pass the small model tests, we need full precision.
49 + for_loop_outputs = []
50 + with vllm_runner(model, dtype=dtype) as vllm_model:
51 + for prompt in example_prompts:
52 + for_loop_outputs.append(
53 + vllm_model.generate_greedy([prompt], max_tokens)[0])
54 +
55 + batched_outputs = vllm_model.generate_greedy(example_prompts,
56 + max_tokens)
57 +
58 + check_outputs_equal(
59 + outputs_0_lst=for_loop_outputs,
60 + outputs_1_lst=batched_outputs,
61 + name_0="for_loop_vllm",
62 + name_1="batched_vllm",
63 + )
64 +
65 +
66 @pytest.mark.parametrize("model", MODELS)
67 @pytest.mark.parametrize("dtype", ["bfloat16"])
68 @pytest.mark.parametrize("max_tokens", [20])
@@ -60,6 +89,60 @@ def test_mamba_cache_cg_padding(
89 "Could be related to mamba cache not padded correctly")
90
91
92 +@pytest.mark.parametrize("model", MODELS)
93 +@pytest.mark.parametrize("dtype", ["float"])
94 +@pytest.mark.parametrize("max_tokens", [20])
95 +def test_models_preemption_recompute(
96 + hf_runner,
97 + vllm_runner,
98 + example_prompts,
99 + model: str,
100 + dtype: str,
101 + max_tokens: int,
102 +) -> None:
103 + # Tests that outputs are identical with and w/o preemtions (recompute)
104 + assert dtype == "float"
105 +
106 + with vllm_runner(model, dtype=dtype) as vllm_model:
107 + vllm_model.model.llm_engine.scheduler[
108 + 0].ENABLE_ARTIFICIAL_PREEMPT = True
109 + preempt_vllm_outputs = vllm_model.generate_greedy(
110 + example_prompts, max_tokens)
111 +
112 + vllm_model.model.llm_engine.scheduler[
113 + 0].ENABLE_ARTIFICIAL_PREEMPT = False
114 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
115 +
116 + check_outputs_equal(
117 + outputs_0_lst=preempt_vllm_outputs,
118 + outputs_1_lst=vllm_outputs,
119 + name_0="vllm_preepmtions",
120 + name_1="vllm",
121 + )
122 +
123 +
124 +@pytest.mark.parametrize("model", MODELS)
125 +@pytest.mark.parametrize("dtype", ["float"])
126 +def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
127 + vllm_runner,
128 + model: str,
129 + dtype: str,
130 + example_prompts,
131 +) -> None:
132 + # This test is for verifying that the Jamba inner state management doesn't
133 + # collapse in case where the number of incoming requests and
134 + # finished_requests_ids is larger than the maximum mamba block capacity.
135 + # This could generally happen due to the fact that Jamba does support
136 + # statelessness mechanism where it can cleanup new incoming requests in
137 + # a single step.
138 + try:
139 + with vllm_runner(model, dtype=dtype, max_num_seqs=10) as vllm_model:
140 + vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
141 + except ValueError:
142 + pytest.fail("Jamba inner state wasn't cleaned up properly between"
143 + "steps finished requests registered unnecessarily ")
144 +
145 +
146 @pytest.mark.parametrize("model", MODELS)
147 @pytest.mark.parametrize("dtype", ["float"])
148 def test_state_cleanup(
vllm/core/scheduler.py
+1
@@ -374,6 +374,7 @@ class Scheduler:
374 for aborted_group in aborted_groups:
375 # Remove the sequence group from the state queue.
376 state_queue.remove(aborted_group)
377 + self._finished_requests_ids.append(aborted_group.request_id)
378 for seq in aborted_group.get_seqs():
379 if seq.is_finished():
380 continue
vllm/model_executor/model_loader/loader.py
+20 -12
@@ -32,7 +32,8 @@ from vllm.model_executor.model_loader.weight_utils import (
32 filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
33 get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34 pt_weights_iterator, safetensors_weights_iterator)
35 -from vllm.model_executor.models.interfaces import (supports_lora,
35 +from vllm.model_executor.models.interfaces import (has_inner_state,
36 + supports_lora,
37 supports_vision)
38 from vllm.model_executor.utils import set_weight_attrs
39 from vllm.platforms import current_platform
@@ -66,10 +67,10 @@ def _get_quantization_config(
67
68
69 def _get_model_initialization_kwargs(
69 - model_class: Type[nn.Module],
70 - lora_config: Optional[LoRAConfig],
71 - multimodal_config: Optional[MultiModalConfig],
72 -) -> Dict[str, Any]:
70 + model_class: Type[nn.Module],
71 + lora_config: Optional[LoRAConfig],
72 + multimodal_config: Optional[MultiModalConfig],
73 + scheduler_config: Optional[SchedulerConfig] = None) -> Dict[str, Any]:
74 """Get extra kwargs for model initialization."""
75 extra_kwargs: Dict[str, Any] = {}
76
@@ -90,13 +91,19 @@ def _get_model_initialization_kwargs(
91
92 extra_kwargs["multimodal_config"] = multimodal_config
93
94 + if has_inner_state(model_class) and scheduler_config:
95 + extra_kwargs["scheduler_config"] = scheduler_config
96 +
97 return extra_kwargs
98
99
96 -def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
97 - lora_config: Optional[LoRAConfig],
98 - multimodal_config: Optional[MultiModalConfig],
99 - cache_config: CacheConfig) -> nn.Module:
100 +def _initialize_model(
101 + model_config: ModelConfig,
102 + load_config: LoadConfig,
103 + lora_config: Optional[LoRAConfig],
104 + multimodal_config: Optional[MultiModalConfig],
105 + cache_config: CacheConfig,
106 + scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
107 """Initialize a model with the given configurations."""
108 model_class = get_model_architecture(model_config)[0]
109 quant_config = _get_quantization_config(model_config, load_config)
@@ -105,7 +112,8 @@ def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
112 cache_config=cache_config,
113 quant_config=quant_config,
114 **_get_model_initialization_kwargs(
108 - model_class, lora_config, multimodal_config))
115 + model_class, lora_config, multimodal_config,
116 + scheduler_config))
117
118
119 class BaseModelLoader(ABC):
@@ -266,7 +274,7 @@ class DefaultModelLoader(BaseModelLoader):
274 with torch.device(device_config.device):
275 model = _initialize_model(model_config, self.load_config,
276 lora_config, multimodal_config,
269 - cache_config)
277 + cache_config, scheduler_config)
278 model.load_weights(
279 self._get_weights_iterator(model_config.model,
280 model_config.revision,
@@ -302,7 +310,7 @@ class DummyModelLoader(BaseModelLoader):
310 with torch.device(device_config.device):
311 model = _initialize_model(model_config, self.load_config,
312 lora_config, multimodal_config,
305 - cache_config)
313 + cache_config, scheduler_config)
314 # NOTE(woosuk): For accurate performance evaluation, we assign
315 # random values to the weights.
316 initialize_dummy_weights(model)
vllm/model_executor/models/interfaces.py
+47 -1
@@ -3,7 +3,7 @@ from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
3
4 from typing_extensions import TypeGuard
5
6 -from vllm.config import LoRAConfig, MultiModalConfig
6 +from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
7 from vllm.logger import init_logger
8
9 logger = init_logger(__name__)
@@ -142,3 +142,49 @@ def _supports_lora(
142 return isinstance(model, _SupportsLoRAType)
143
144 return isinstance(model, SupportsLoRA)
145 +
146 +
147 +@runtime_checkable
148 +class HasInnerState(Protocol):
149 + """The interface required for all models that has inner state."""
150 +
151 + has_inner_state: ClassVar[Literal[True]] = True
152 + """
153 + A flag that indicates this model has inner state.
154 + Models that has inner state usually need access to the scheduler_config
155 + for max_num_seqs ,etc... (Currently only used by Jamba)
156 + """
157 +
158 + def __init__(self,
159 + *,
160 + scheduler_config: Optional[SchedulerConfig] = None) -> None:
161 + ...
162 +
163 +
164 +@runtime_checkable
165 +class _HasInnerStateType(Protocol):
166 + has_inner_state: ClassVar[Literal[True]]
167 +
168 + def __init__(self,
169 + *,
170 + scheduler_config: Optional[SchedulerConfig] = None) -> None:
171 + ...
172 +
173 +
174 +@overload
175 +def has_inner_state(model: object) -> TypeGuard[HasInnerState]:
176 + ...
177 +
178 +
179 +@overload
180 +def has_inner_state(model: Type[object]) -> TypeGuard[Type[HasInnerState]]:
181 + ...
182 +
183 +
184 +def has_inner_state(
185 + model: Union[Type[object], object]
186 +) -> Union[TypeGuard[Type[HasInnerState]], TypeGuard[HasInnerState]]:
187 + if isinstance(model, type):
188 + return isinstance(model, _HasInnerStateType)
189 +
190 + return isinstance(model, HasInnerState)
vllm/model_executor/models/jamba.py
+25 -11
@@ -13,7 +13,7 @@ from transformers import JambaConfig
13
14 from vllm.attention.backends.abstract import AttentionMetadata
15 from vllm.attention.layer import Attention
16 -from vllm.config import CacheConfig, LoRAConfig
16 +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
17 from vllm.distributed import (get_tensor_model_parallel_rank,
18 get_tensor_model_parallel_world_size,
19 tensor_model_parallel_all_reduce)
@@ -32,10 +32,12 @@ from vllm.model_executor.layers.sampler import Sampler
32 from vllm.model_executor.layers.vocab_parallel_embedding import (
33 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
34 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35 +from vllm.model_executor.models.interfaces import HasInnerState
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.model_executor.utils import set_weight_attrs
38 from vllm.sequence import IntermediateTensors, SamplerOutput
38 -from vllm.worker.model_runner import _BATCH_SIZES_TO_CAPTURE
39 +from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
40 + _get_graph_batch_size)
41
42 KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -612,7 +614,7 @@ class JambaModel(nn.Module):
614 return hidden_states
615
616
615 -class JambaForCausalLM(nn.Module):
617 +class JambaForCausalLM(nn.Module, HasInnerState):
618 packed_modules_mapping = {
619 "qkv_proj": [
620 "q_proj",
@@ -640,9 +642,11 @@ class JambaForCausalLM(nn.Module):
642 cache_config: Optional[CacheConfig] = None,
643 quant_config: Optional[QuantizationConfig] = None,
644 lora_config: Optional[LoRAConfig] = None,
645 + scheduler_config: Optional[SchedulerConfig] = None,
646 ) -> None:
647 super().__init__()
648 self.config = config
649 + self.scheduler_config = scheduler_config
650 self.model = JambaModel(config,
651 cache_config=cache_config,
652 quant_config=quant_config,
@@ -689,6 +693,8 @@ class JambaForCausalLM(nn.Module):
693 for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
694
695 request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
696 + finished_requests_ids = kwargs["finished_requests_ids"]
697 + self._release_mamba_cache(finished_requests_ids)
698 batch_size = input_ids.shape[0]
699 if attn_metadata.prefill_metadata:
700 batch_size = len(request_ids_to_seq_ids)
@@ -696,9 +702,8 @@ class JambaForCausalLM(nn.Module):
702 current_seqlen_agnostic_cache,
703 indices,
704 ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
699 - batch_size)
700 - finished_requests_ids = kwargs["finished_requests_ids"]
701 - self._release_mamba_cache(finished_requests_ids)
705 + batch_size,
706 + finished_requests_ids)
707 else:
708 # CUDA graph capturing runs
709 current_seqlen_agnostic_cache, indices = (
@@ -760,10 +765,15 @@ class JambaForCausalLM(nn.Module):
765 return indices_for_current_run
766
767 def _prepare_current_run_mamba_cache(
763 - self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int
768 + self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int,
769 + finished_requests_ids: List[str]
770 ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], List[int]]:
771 indices_for_current_run = []
772 for request_id, seqs_id in request_ids_to_seq_ids.items():
773 + if request_id in finished_requests_ids:
774 + # Do not allocate cache for requests that run
775 + # and finish right after
776 + continue
777 indices_for_current_run += self._assign_seq_id_to_mamba_cache(
778 request_id, seqs_id)
779 ## Pad the batch in case of running batch that was not captured via CG
@@ -787,16 +797,17 @@ class JambaForCausalLM(nn.Module):
797 assert all(
798 key in kwargs
799 for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
800 + finished_requests_ids = kwargs["finished_requests_ids"]
801 + self._release_mamba_cache(finished_requests_ids)
802 request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
803 cg_batch_size = input_buffers['input_ids'].shape[0]
804 (
805 current_mamba_cache,
806 indices,
807 ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796 - cg_batch_size)
808 + cg_batch_size,
809 + finished_requests_ids)
810 self.current_indices = indices
798 - finished_requests_ids = kwargs["finished_requests_ids"]
799 - self._release_mamba_cache(finished_requests_ids)
811
812 for input_buffer, current_cache_buffer in zip(
813 input_buffers["seqlen_agnostic_capture_inputs"],
@@ -860,9 +871,12 @@ class JambaForCausalLM(nn.Module):
871 layers_type = self.config.layers_block_type
872 mamba_layers = sum(
873 [layer_type == "mamba" for layer_type in layers_type])
863 - max_batch_size = _BATCH_SIZES_TO_CAPTURE[-1] + 10
874 + max_batch_size = (_get_graph_batch_size(
875 + self.scheduler_config.max_num_seqs) if self.scheduler_config else
876 + max(_BATCH_SIZES_TO_CAPTURE)) + 10
877 conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
878 assert conv_state_shape is not None and temporal_state_shape is not None
879 +
880 for buffername in ["mamba_cache", "mamba_gc_cache_buffer"]:
881 buffer = (torch.empty(size=(mamba_layers, max_batch_size) +
882 conv_state_shape,