A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
Support mistral interleaved attn (#9414)
Patrick von Platen committed
Oct 16, 2024 at 15:28 UTC
415f76a9cbcdec9346661e5b6f04c35a4d8eb3f4
1 file changed
+28
-10
vllm/config.py
+28
-10
@@ -173,14 +173,20 @@ class ModelConfig:
173
if self.enforce_eager is None:
174
self.enforce_eager = False
175
176
- if (not self.disable_sliding_window
177
- and self.hf_text_config.model_type == "gemma2"
178
- and self.hf_text_config.sliding_window is not None):
176
+ sliding_window = getattr(self.hf_text_config, "sliding_window", None)
177
+ has_interleaved_attention = (sliding_window is not None) and (
178
+ isinstance(sliding_window, list) or
179
+ (self.hf_text_config.model_type in ["gemma2"]))
180
+
181
+ if (not self.disable_sliding_window and has_interleaved_attention):
182
+ sliding_window_len_min = get_min_sliding_window(
183
+ self.hf_text_config.sliding_window)
184
+
185
print_warning_once(
180
- "Gemma 2 uses sliding window attention for every odd layer, "
186
+ f"{self.hf_text_config.model_type} has interleaved attention, "
187
"which is currently not supported by vLLM. Disabling sliding "
188
"window and capping the max length to the sliding window size "
183
- f"({self.hf_text_config.sliding_window}).")
189
+ f"({sliding_window_len_min}).")
190
self.disable_sliding_window = True
191
192
self.max_model_len = _get_and_verify_max_len(
@@ -431,7 +437,8 @@ class ModelConfig:
437
"pipeline parallelism currently. Disabling it.")
438
self.use_async_output_proc = False
439
434
- def get_hf_config_sliding_window(self) -> Optional[int]:
440
+ def get_hf_config_sliding_window(
441
+ self) -> Union[Optional[int], List[Optional[int]]]:
442
"""Get the sliding window size, or None if disabled."""
443
444
# Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
@@ -442,7 +449,7 @@ class ModelConfig:
449
return None
450
return getattr(self.hf_text_config, "sliding_window", None)
451
445
- def get_sliding_window(self) -> Optional[int]:
452
+ def get_sliding_window(self) -> Optional[Union[int, List[Optional[int]]]]:
453
"""Get the sliding window size, or None if disabled.
454
"""
455
# If user disables sliding window, return None.
@@ -1689,7 +1696,7 @@ def _get_and_verify_max_len(
1696
hf_config: PretrainedConfig,
1697
max_model_len: Optional[int],
1698
disable_sliding_window: bool,
1692
- sliding_window_len: Optional[int],
1699
+ sliding_window_len: Optional[Union[int, List[Optional[int]]]],
1700
spec_target_max_model_len: Optional[int] = None,
1701
) -> int:
1702
"""Get and verify the model's maximum length."""
@@ -1722,9 +1729,12 @@ def _get_and_verify_max_len(
1729
# If sliding window is manually disabled, max_length should be less
1730
# than the sliding window length in the model config.
1731
if disable_sliding_window and sliding_window_len is not None:
1732
+
1733
+ sliding_window_len_min = get_min_sliding_window(sliding_window_len)
1734
max_len_key = "sliding_window" \
1726
- if sliding_window_len < derived_max_model_len else max_len_key
1727
- derived_max_model_len = min(derived_max_model_len, sliding_window_len)
1735
+ if sliding_window_len_min < derived_max_model_len else max_len_key
1736
+ derived_max_model_len = min(derived_max_model_len,
1737
+ sliding_window_len_min)
1738
1739
# If none of the keys were found in the config, use a default and
1740
# log a warning.
@@ -1805,6 +1815,14 @@ def _get_and_verify_max_len(
1815
return int(max_model_len)
1816
1817
1818
+def get_min_sliding_window(
1819
+ sliding_window: Union[int, List[Optional[int]]]) -> int:
1820
+ if isinstance(sliding_window, list):
1821
+ return min(s for s in sliding_window if s is not None)
1822
+
1823
+ return sliding_window
1824
+
1825
+
1826
def get_served_model_name(model: str,
1827
served_model_name: Optional[Union[str, List[str]]]):
1828
"""