@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1102bef219
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix / Core] Prefix Caching Guards (merged with main) (#4846)

Co-authored-by: rsnm2 <rshaw@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

Zhuohan Li committed May 27, 2024 at 15:18 UTC 1102bef2195a102c6a5489a28329e543a600b4d8
11 files changed +167 -44
tests/prefix_caching/test_disable_sliding_window.py new
+44
@@ -0,0 +1,44 @@
1 +"""Compare the with and without prefix caching.
2 +
3 +Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 +"""
5 +import pytest
6 +
7 +from tests.conftest import cleanup
8 +from vllm import LLM
9 +
10 +MODEL_LEN_LEN = [
11 + # Example models with sliding window.
12 + ("bigcode/starcoder2-3b", 4096, 16384),
13 + # ("mistralai/Mistral-7B-v0.1", 4096, 32768), << OOM in CI
14 +
15 + # Confirm model with sliding window works.
16 + # config has "use_sliding_window": false
17 + ("Qwen/Qwen1.5-0.5B-Chat", 32768, 32768),
18 + # config has no sliding window attribute.
19 + ("TinyLlama/TinyLlama-1.1B-Chat-v1.0", 2048, 2048),
20 +]
21 +
22 +
23 +@pytest.mark.parametrize("model_len_len", MODEL_LEN_LEN)
24 +def test_disable_sliding_window(model_len_len, ):
25 + model, sliding_len, full_len = model_len_len
26 + vllm_disabled_model = LLM(model, disable_sliding_window=True)
27 + vllm_disabled_model.generate("Hi my name is")
28 + model_config = vllm_disabled_model.llm_engine.model_config
29 + assert model_config.max_model_len == sliding_len, (
30 + "Max len expected to equal sliding_len of %s, but got %s", sliding_len,
31 + model_config.max_model_len)
32 +
33 + del vllm_disabled_model
34 + cleanup()
35 +
36 + vllm_enabled_model = LLM(model, disable_sliding_window=False)
37 + vllm_enabled_model.generate("Hi my name is")
38 + model_config = vllm_enabled_model.llm_engine.model_config
39 + assert model_config.max_model_len == full_len, (
40 + "Max len expected to equal full_len of %s, but got %s", full_len,
41 + model_config.max_model_len)
42 +
43 + del vllm_enabled_model
44 + cleanup()
tests/test_config.py
+24
@@ -1,5 +1,29 @@
1 +import pytest
2 +
3 from vllm.config import ModelConfig
4
5 +MODEL_IDS_EXPECTED = [
6 + ("Qwen/Qwen1.5-7B", 32768),
7 + ("mistralai/Mistral-7B-v0.1", 4096),
8 + ("mistralai/Mistral-7B-Instruct-v0.2", 32768),
9 +]
10 +
11 +
12 +@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
13 +def test_disable_sliding_window(model_id_expected):
14 + model_id, expected = model_id_expected
15 + model_config = ModelConfig(
16 + model_id,
17 + model_id,
18 + tokenizer_mode="auto",
19 + trust_remote_code=False,
20 + seed=0,
21 + dtype="float16",
22 + revision=None,
23 + disable_sliding_window=True,
24 + )
25 + assert model_config.max_model_len == expected
26 +
27
28 def test_get_sliding_window():
29 TEST_SLIDING_WINDOW = 4096
vllm/attention/layer.py
+2 -1
@@ -30,7 +30,6 @@ class Attention(nn.Module):
30 scale: float,
31 num_kv_heads: Optional[int] = None,
32 alibi_slopes: Optional[List[float]] = None,
33 - sliding_window: Optional[int] = None,
33 cache_config: Optional[CacheConfig] = None,
34 quant_config: Optional[QuantizationConfig] = None,
35 blocksparse_params: Optional[Dict[str, Any]] = None,
@@ -39,9 +38,11 @@ class Attention(nn.Module):
38 if cache_config is not None:
39 kv_cache_dtype = cache_config.cache_dtype
40 block_size = cache_config.block_size
41 + sliding_window = cache_config.sliding_window
42 else:
43 kv_cache_dtype = "auto"
44 block_size = 16
45 + sliding_window = None
46 if num_kv_heads is None:
47 num_kv_heads = num_heads
48
vllm/config.py
+64 -3
@@ -69,6 +69,10 @@ class ModelConfig:
69 max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
70 When a sequence has context length larger than this, we fall back
71 to eager mode
72 + disable_sliding_window: Whether to disable sliding window. If True,
73 + we will disable the sliding window functionality of the model.
74 + If the model does not support sliding window, this argument is
75 + ignored.
76 skip_tokenizer_init: If true, skip initialization of tokenizer and
77 detokenizer.
78 served_model_name: The model name used in metrics tag `model_name`,
@@ -96,6 +100,7 @@ class ModelConfig:
100 max_context_len_to_capture: Optional[int] = None,
101 max_seq_len_to_capture: Optional[int] = None,
102 max_logprobs: int = 5,
103 + disable_sliding_window: bool = False,
104 skip_tokenizer_init: bool = False,
105 served_model_name: Optional[Union[str, List[str]]] = None,
106 ) -> None:
@@ -118,14 +123,18 @@ class ModelConfig:
123 self.max_seq_len_to_capture = (max_seq_len_to_capture
124 or max_context_len_to_capture)
125 self.max_logprobs = max_logprobs
126 + self.disable_sliding_window = disable_sliding_window
127 self.skip_tokenizer_init = skip_tokenizer_init
128
129 self.hf_config = get_config(self.model, trust_remote_code, revision,
130 code_revision, rope_scaling)
131 self.hf_text_config = get_hf_text_config(self.hf_config)
132 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
127 - self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
128 - max_model_len)
133 + self.max_model_len = _get_and_verify_max_len(
134 + hf_config=self.hf_text_config,
135 + max_model_len=max_model_len,
136 + disable_sliding_window=self.disable_sliding_window,
137 + sliding_window_len=self.get_hf_config_sliding_window())
138 self.served_model_name = get_served_model_name(model,
139 served_model_name)
140 if not self.skip_tokenizer_init:
@@ -220,7 +229,7 @@ class ModelConfig:
229 "must be divisible by pipeline parallel size "
230 f"({pipeline_parallel_size}).")
231
223 - def get_sliding_window(self) -> Optional[int]:
232 + def get_hf_config_sliding_window(self) -> Optional[int]:
233 """Get the sliding window size, or None if disabled.
234 """
235
@@ -232,6 +241,15 @@ class ModelConfig:
241 return None
242 return getattr(self.hf_text_config, "sliding_window", None)
243
244 + def get_sliding_window(self) -> Optional[int]:
245 + """Get the sliding window size, or None if disabled.
246 + """
247 + # If user disables sliding window, return None.
248 + if self.disable_sliding_window:
249 + return None
250 + # Otherwise get the value from the hf config.
251 + return self.get_hf_config_sliding_window()
252 +
253 def get_vocab_size(self) -> int:
254 return self.hf_text_config.vocab_size
255
@@ -336,6 +354,7 @@ class CacheConfig:
354 self.enable_prefix_caching = enable_prefix_caching
355 self._verify_args()
356 self._verify_cache_dtype()
357 + self._verify_prefix_caching()
358
359 # Will be set after profiling.
360 self.num_gpu_blocks = None
@@ -364,6 +383,19 @@ class CacheConfig:
383 else:
384 raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
385
386 + def _verify_prefix_caching(self) -> None:
387 + if not self.enable_prefix_caching:
388 + return
389 +
390 + if self.sliding_window is not None:
391 + raise NotImplementedError(
392 + "Prefix caching is not supported with sliding window. "
393 + "Run with --disable-sliding-window to use prefix caching.")
394 + if self.cache_dtype == "fp8":
395 + raise NotImplementedError(
396 + "Prefix caching is not supported for fp8 cache_dtype. "
397 + "Run with --kv-cache-dtype auto to use prefix caching.")
398 +
399 def verify_with_parallel_config(
400 self,
401 parallel_config: "ParallelConfig",
@@ -1116,6 +1148,8 @@ def _get_and_verify_dtype(
1148 def _get_and_verify_max_len(
1149 hf_config: PretrainedConfig,
1150 max_model_len: Optional[int],
1151 + disable_sliding_window: bool,
1152 + sliding_window_len: Optional[int],
1153 ) -> int:
1154 """Get and verify the model's maximum length."""
1155 derived_max_model_len = float("inf")
@@ -1135,6 +1169,7 @@ def _get_and_verify_max_len(
1169 "max_seq_length",
1170 "seq_len",
1171 ]
1172 + # Choose the smallest "max_length" from the possible keys.
1173 max_len_key = None
1174 for key in possible_keys:
1175 max_len = getattr(hf_config, key, None)
@@ -1142,6 +1177,16 @@ def _get_and_verify_max_len(
1177 max_len_key = key if max_len < derived_max_model_len \
1178 else max_len_key
1179 derived_max_model_len = min(derived_max_model_len, max_len)
1180 +
1181 + # If sliding window is manually disabled, max_length should be less
1182 + # than the sliding window length in the model config.
1183 + if disable_sliding_window and sliding_window_len is not None:
1184 + max_len_key = "sliding_window" \
1185 + if sliding_window_len < derived_max_model_len else max_len_key
1186 + derived_max_model_len = min(derived_max_model_len, sliding_window_len)
1187 +
1188 + # If none of the keys were found in the config, use a default and
1189 + # log a warning.
1190 if derived_max_model_len == float("inf"):
1191 if max_model_len is not None:
1192 # If max_model_len is specified, we use it.
@@ -1157,6 +1202,13 @@ def _get_and_verify_max_len(
1202
1203 rope_scaling = getattr(hf_config, "rope_scaling", None)
1204 if rope_scaling is not None and rope_scaling["type"] != "su":
1205 + if disable_sliding_window:
1206 + # TODO(robertgshaw): Find a model that supports rope_scaling
1207 + # with sliding window to see if this case should be allowed.
1208 + raise NotImplementedError(
1209 + "Disabling sliding window is not supported for models "
1210 + "with rope_scaling. Please raise an issue so we can "
1211 + "investigate.")
1212 assert "factor" in rope_scaling
1213 scaling_factor = rope_scaling["factor"]
1214 if rope_scaling["type"] == "yarn":
@@ -1164,6 +1216,8 @@ def _get_and_verify_max_len(
1216 "original_max_position_embeddings"]
1217 derived_max_model_len *= scaling_factor
1218
1219 + # If the user specified a max length, make sure it is smaller than the
1220 + # derived length from the HF model config.
1221 if max_model_len is None:
1222 max_model_len = int(derived_max_model_len)
1223 elif max_model_len > derived_max_model_len:
@@ -1172,6 +1226,13 @@ def _get_and_verify_max_len(
1226 # with model_max_length and allow this override when it's smaller.
1227 model_max_length = getattr(hf_config, "model_max_length", None)
1228 if model_max_length is not None and max_model_len <= model_max_length:
1229 + if disable_sliding_window:
1230 + # TODO(robertgshaw): Find a model that has model_max_length
1231 + # with sliding window to see if this case should be allowed.
1232 + raise NotImplementedError(
1233 + "Disabling sliding window is not supported for models "
1234 + "model_max_length in the config. Please raise an issue "
1235 + "so we can investigate.")
1236 pass
1237 else:
1238 raise ValueError(
vllm/engine/arg_utils.py
+9 -3
@@ -41,6 +41,7 @@ class EngineArgs:
41 max_parallel_loading_workers: Optional[int] = None
42 block_size: int = 16
43 enable_prefix_caching: bool = False
44 + disable_sliding_window: bool = False
45 use_v2_block_manager: bool = False
46 swap_space: int = 4 # GiB
47 gpu_memory_utilization: float = 0.90
@@ -267,6 +268,10 @@ class EngineArgs:
268 parser.add_argument('--enable-prefix-caching',
269 action='store_true',
270 help='Enables automatic prefix caching.')
271 + parser.add_argument('--disable-sliding-window',
272 + action='store_true',
273 + help='Disables sliding window, '
274 + 'capping to sliding window size')
275 parser.add_argument('--use-v2-block-manager',
276 action='store_true',
277 help='Use BlockSpaceMangerV2.')
@@ -558,8 +563,8 @@ class EngineArgs:
563 self.max_model_len, self.quantization,
564 self.quantization_param_path, self.enforce_eager,
565 self.max_context_len_to_capture, self.max_seq_len_to_capture,
561 - self.max_logprobs, self.skip_tokenizer_init,
562 - self.served_model_name)
566 + self.max_logprobs, self.disable_sliding_window,
567 + self.skip_tokenizer_init, self.served_model_name)
568 cache_config = CacheConfig(self.block_size,
569 self.gpu_memory_utilization,
570 self.swap_space, self.kv_cache_dtype,
@@ -645,7 +650,8 @@ class EngineArgs:
650 if (model_config.get_sliding_window() is not None
651 and scheduler_config.chunked_prefill_enabled):
652 raise ValueError(
648 - "Chunked prefill is not supported with sliding window.")
653 + "Chunked prefill is not supported with sliding window. "
654 + "Set --disable-sliding-window to disable sliding window.")
655
656 return EngineConfig(model_config=model_config,
657 cache_config=cache_config,
vllm/model_executor/models/llama.py
-4
@@ -94,7 +94,6 @@ class LlamaAttention(nn.Module):
94 max_position_embeddings: int = 8192,
95 quant_config: Optional[QuantizationConfig] = None,
96 bias: bool = False,
97 - sliding_window: Optional[int] = None,
97 cache_config: Optional[CacheConfig] = None,
98 ) -> None:
99 super().__init__()
@@ -146,7 +145,6 @@ class LlamaAttention(nn.Module):
145 self.head_dim,
146 self.scaling,
147 num_kv_heads=self.num_kv_heads,
149 - sliding_window=sliding_window,
148 cache_config=cache_config,
149 quant_config=quant_config)
150
@@ -183,7 +181,6 @@ class LlamaDecoderLayer(nn.Module):
181 config.original_max_position_embeddings)
182 max_position_embeddings = getattr(config, "max_position_embeddings",
183 8192)
186 - sliding_window = getattr(config, "sliding_window", None)
184 # Support abacusai/Smaug-72B-v0.1 with attention_bias
185 # Support internlm/internlm-7b with bias
186 attention_bias = getattr(config, "attention_bias", False) or getattr(
@@ -198,7 +195,6 @@ class LlamaDecoderLayer(nn.Module):
195 max_position_embeddings=max_position_embeddings,
196 quant_config=quant_config,
197 bias=attention_bias,
201 - sliding_window=sliding_window,
198 cache_config=cache_config,
199 )
200 self.mlp = LlamaMLP(
vllm/model_executor/models/mixtral.py
+10 -12
@@ -246,15 +246,16 @@ class MixtralMoE(nn.Module):
246
247 class MixtralAttention(nn.Module):
248
249 - def __init__(self,
250 - hidden_size: int,
251 - num_heads: int,
252 - num_kv_heads: int,
253 - max_position: int = 4096 * 32,
254 - rope_theta: float = 10000,
255 - cache_config: Optional[CacheConfig] = None,
256 - quant_config: Optional[QuantizationConfig] = None,
257 - sliding_window: Optional[int] = None) -> None:
249 + def __init__(
250 + self,
251 + hidden_size: int,
252 + num_heads: int,
253 + num_kv_heads: int,
254 + max_position: int = 4096 * 32,
255 + rope_theta: float = 10000,
256 + cache_config: Optional[CacheConfig] = None,
257 + quant_config: Optional[QuantizationConfig] = None,
258 + ) -> None:
259 super().__init__()
260 self.hidden_size = hidden_size
261 tp_size = get_tensor_model_parallel_world_size()
@@ -276,7 +277,6 @@ class MixtralAttention(nn.Module):
277 self.kv_size = self.num_kv_heads * self.head_dim
278 self.scaling = self.head_dim**-0.5
279 self.rope_theta = rope_theta
279 - self.sliding_window = sliding_window
280
281 if isinstance(
282 quant_config,
@@ -312,7 +312,6 @@ class MixtralAttention(nn.Module):
312 self.head_dim,
313 self.scaling,
314 num_kv_heads=self.num_kv_heads,
315 - sliding_window=self.sliding_window,
315 cache_config=cache_config,
316 quant_config=quant_config)
317
@@ -349,7 +348,6 @@ class MixtralDecoderLayer(nn.Module):
348 max_position=config.max_position_embeddings,
349 num_kv_heads=config.num_key_value_heads,
350 rope_theta=rope_theta,
352 - sliding_window=config.sliding_window,
351 cache_config=cache_config,
352 quant_config=quant_config)
353 self.block_sparse_moe = MixtralMoE(
vllm/model_executor/models/mixtral_quant.py
-4
@@ -166,7 +166,6 @@ class MixtralAttention(nn.Module):
166 max_position: int = 4096 * 32,
167 rope_theta: float = 10000,
168 quant_config: Optional[QuantizationConfig] = None,
169 - sliding_window: Optional[int] = None,
169 cache_config: Optional[CacheConfig] = None,
170 ) -> None:
171 super().__init__()
@@ -190,7 +189,6 @@ class MixtralAttention(nn.Module):
189 self.kv_size = self.num_kv_heads * self.head_dim
190 self.scaling = self.head_dim**-0.5
191 self.rope_theta = rope_theta
193 - self.sliding_window = sliding_window
192
193 self.qkv_proj = QKVParallelLinear(
194 hidden_size,
@@ -217,7 +215,6 @@ class MixtralAttention(nn.Module):
215 self.head_dim,
216 self.scaling,
217 num_kv_heads=self.num_kv_heads,
220 - sliding_window=self.sliding_window,
218 cache_config=cache_config,
219 quant_config=quant_config)
220
@@ -254,7 +251,6 @@ class MixtralDecoderLayer(nn.Module):
251 max_position=config.max_position_embeddings,
252 num_kv_heads=config.num_key_value_heads,
253 rope_theta=rope_theta,
257 - sliding_window=config.sliding_window,
254 cache_config=cache_config,
255 quant_config=quant_config)
256 self.block_sparse_moe = MixtralMoE(config=config,
vllm/model_executor/models/qwen2.py
+14 -11
@@ -86,10 +86,8 @@ class Qwen2Attention(nn.Module):
86 num_kv_heads: int,
87 max_position: int = 4096 * 32,
88 rope_theta: float = 10000,
89 - use_sliding_window: bool = False,
89 cache_config: Optional[CacheConfig] = None,
90 quant_config: Optional[QuantizationConfig] = None,
92 - sliding_window: Optional[int] = None,
91 rope_scaling: Optional[Tuple] = None) -> None:
92 super().__init__()
93 self.hidden_size = hidden_size
@@ -112,7 +110,6 @@ class Qwen2Attention(nn.Module):
110 self.kv_size = self.num_kv_heads * self.head_dim
111 self.scaling = self.head_dim**-0.5
112 self.rope_theta = rope_theta
115 - self.sliding_window = sliding_window if use_sliding_window else None
113
114 self.qkv_proj = QKVParallelLinear(
115 hidden_size,
@@ -140,7 +137,6 @@ class Qwen2Attention(nn.Module):
137 self.head_dim,
138 self.scaling,
139 num_kv_heads=self.num_kv_heads,
143 - sliding_window=self.sliding_window,
140 cache_config=cache_config,
141 quant_config=quant_config)
142
@@ -164,7 +160,6 @@ class Qwen2DecoderLayer(nn.Module):
160 def __init__(
161 self,
162 config: Qwen2Config,
167 - layer_idx: int,
163 cache_config: Optional[CacheConfig] = None,
164 quant_config: Optional[QuantizationConfig] = None,
165 ) -> None:
@@ -173,18 +168,14 @@ class Qwen2DecoderLayer(nn.Module):
168 # Requires transformers > 4.32.0
169 rope_theta = getattr(config, "rope_theta", 1000000)
170 rope_scaling = getattr(config, "rope_scaling", None)
176 - use_sliding_window = (config.use_sliding_window
177 - and layer_idx < config.max_window_layers)
171 self.self_attn = Qwen2Attention(
172 hidden_size=self.hidden_size,
173 num_heads=config.num_attention_heads,
174 max_position=config.max_position_embeddings,
175 num_kv_heads=config.num_key_value_heads,
176 rope_theta=rope_theta,
184 - use_sliding_window=use_sliding_window,
177 cache_config=cache_config,
178 quant_config=quant_config,
187 - sliding_window=config.sliding_window,
179 rope_scaling=rope_scaling)
180 self.mlp = Qwen2MLP(
181 hidden_size=self.hidden_size,
@@ -244,8 +235,8 @@ class Qwen2Model(nn.Module):
235 config.hidden_size,
236 )
237 self.layers = nn.ModuleList([
247 - Qwen2DecoderLayer(config, layer_idx, cache_config, quant_config)
248 - for layer_idx in range(config.num_hidden_layers)
238 + Qwen2DecoderLayer(config, cache_config, quant_config)
239 + for _ in range(config.num_hidden_layers)
240 ])
241 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
242
@@ -302,6 +293,18 @@ class Qwen2ForCausalLM(nn.Module):
293 lora_config: Optional[LoRAConfig] = None,
294 ) -> None:
295 del lora_config
296 + # TODO (@robertgshaw2): see if this can be moved out
297 + if (cache_config.sliding_window is not None
298 + and hasattr(config, "max_window_layers")):
299 + raise ValueError("Sliding window for some but all layers is not "
300 + "supported. This model uses sliding window "
301 + "but `max_window_layers` = %s is less than "
302 + "`num_hidden_layers` = %s. Please open an issue "
303 + "to discuss this feature." % (
304 + config.max_window_layers,
305 + config.num_hidden_layers,
306 + ))
307 +
308 super().__init__()
309 self.config = config
310 self.quant_config = quant_config
vllm/model_executor/models/starcoder2.py
-2
@@ -74,7 +74,6 @@ class Starcoder2Attention(nn.Module):
74 self.rope_theta = config.rope_theta
75 self.max_position_embeddings = config.max_position_embeddings
76 self.use_bias = config.use_bias
77 - self.sliding_window = config.sliding_window
77
78 self.qkv_proj = QKVParallelLinear(
79 self.hidden_size,
@@ -101,7 +100,6 @@ class Starcoder2Attention(nn.Module):
100 self.head_dim,
101 self.scaling,
102 num_kv_heads=self.num_kv_heads,
104 - sliding_window=self.sliding_window,
103 cache_config=cache_config,
104 quant_config=quant_config)
105
vllm/model_executor/models/xverse.py
-4
@@ -88,7 +88,6 @@ class XverseAttention(nn.Module):
88 max_position_embeddings: int = 8192,
89 quant_config: Optional[QuantizationConfig] = None,
90 bias: bool = False,
91 - sliding_window: Optional[int] = None,
91 cache_config: Optional[CacheConfig] = None,
92 ) -> None:
93 super().__init__()
@@ -134,7 +133,6 @@ class XverseAttention(nn.Module):
133 self.head_dim,
134 self.scaling,
135 num_kv_heads=self.num_kv_heads,
137 - sliding_window=sliding_window,
136 cache_config=cache_config,
137 quant_config=quant_config)
138
@@ -167,7 +165,6 @@ class XverseDecoderLayer(nn.Module):
165 rope_scaling = getattr(config, "rope_scaling", None)
166 max_position_embeddings = getattr(config, "max_position_embeddings",
167 8192)
170 - sliding_window = getattr(config, "sliding_window", None)
168 self.self_attn = XverseAttention(
169 hidden_size=self.hidden_size,
170 num_heads=config.num_attention_heads,
@@ -178,7 +175,6 @@ class XverseDecoderLayer(nn.Module):
175 max_position_embeddings=max_position_embeddings,
176 quant_config=quant_config,
177 bias=getattr(config, "bias", False),
181 - sliding_window=sliding_window,
178 cache_config=cache_config,
179 )
180 self.mlp = XverseMLP(