A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix assertion failure in Qwen 1.5 with prefix caching enabled (#3373)
Co-authored-by: Cade Daniel <edacih@gmail.com>
陈序 committed
Mar 15, 2024 at 04:56 UTC
54be8a0be2819340ce7c2d7993382559597f5665
2 files changed
+55
-2
tests/test_config.py
new
+43
@@ -0,0 +1,43 @@
1
+from vllm.config import ModelConfig
2
+
3
+
4
+def test_get_sliding_window():
5
+ TEST_SLIDING_WINDOW = 4096
6
+ # Test that the sliding window is correctly computed.
7
+ # For Qwen1.5/Qwen2, get_sliding_window() should be None
8
+ # when use_sliding_window is False.
9
+ qwen2_model_config = ModelConfig(
10
+ "Qwen/Qwen1.5-7B",
11
+ "Qwen/Qwen1.5-7B",
12
+ tokenizer_mode="auto",
13
+ trust_remote_code=False,
14
+ download_dir=None,
15
+ load_format="dummy",
16
+ seed=0,
17
+ dtype="float16",
18
+ revision=None,
19
+ )
20
+
21
+ qwen2_model_config.hf_config.use_sliding_window = False
22
+ qwen2_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
23
+ assert qwen2_model_config.get_sliding_window() is None
24
+
25
+ qwen2_model_config.hf_config.use_sliding_window = True
26
+ assert qwen2_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
27
+
28
+ mistral_model_config = ModelConfig(
29
+ "mistralai/Mistral-7B-v0.1",
30
+ "mistralai/Mistral-7B-v0.1",
31
+ tokenizer_mode="auto",
32
+ trust_remote_code=False,
33
+ download_dir=None,
34
+ load_format="dummy",
35
+ seed=0,
36
+ dtype="float16",
37
+ revision=None,
38
+ )
39
+ mistral_model_config.hf_config.sliding_window = None
40
+ assert mistral_model_config.get_sliding_window() is None
41
+
42
+ mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
43
+ assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
\ No newline at end of file
vllm/config.py
+12
-2
@@ -103,6 +103,7 @@ class ModelConfig:
103
# download model from ModelScope hub,
104
# lazy import so that modelscope is not required for normal use.
105
from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=C
106
+
107
if not os.path.exists(model):
108
model_path = snapshot_download(model_id=model,
109
cache_dir=download_dir,
@@ -139,7 +140,7 @@ class ModelConfig:
140
if (f not in rocm_not_supported_load_format)
141
]
142
raise ValueError(
142
- f"load format \'{load_format}\' is not supported in ROCm. "
143
+ f"load format '{load_format}' is not supported in ROCm. "
144
f"Supported load format are "
145
f"{rocm_supported_load_format}")
146
@@ -232,6 +233,15 @@ class ModelConfig:
233
f"({pipeline_parallel_size}).")
234
235
def get_sliding_window(self) -> Optional[int]:
236
+ """Get the sliding window size, or None if disabled.
237
+ """
238
+
239
+ # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
240
+ # addition to sliding window size. We check if that field is present
241
+ # and if it's False, return None.
242
+ if (hasattr(self.hf_config, "use_sliding_window")
243
+ and not self.hf_config.use_sliding_window):
244
+ return None
245
return getattr(self.hf_config, "sliding_window", None)
246
247
def get_vocab_size(self) -> int:
@@ -624,7 +634,7 @@ def _get_and_verify_dtype(
634
k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
635
if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
636
]
627
- raise ValueError(f"dtype \'{dtype}\' is not supported in ROCm. "
637
+ raise ValueError(f"dtype '{dtype}' is not supported in ROCm. "
638
f"Supported dtypes are {rocm_supported_dtypes}")
639
640
# Verify the dtype.