@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54be8a0be2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix assertion failure in Qwen 1.5 with prefix caching enabled (#3373)

Co-authored-by: Cade Daniel <edacih@gmail.com>

陈序 committed Mar 15, 2024 at 04:56 UTC 54be8a0be2819340ce7c2d7993382559597f5665
2 files changed +55 -2
tests/test_config.py new
+43
@@ -0,0 +1,43 @@
1 +from vllm.config import ModelConfig
2 +
3 +
4 +def test_get_sliding_window():
5 + TEST_SLIDING_WINDOW = 4096
6 + # Test that the sliding window is correctly computed.
7 + # For Qwen1.5/Qwen2, get_sliding_window() should be None
8 + # when use_sliding_window is False.
9 + qwen2_model_config = ModelConfig(
10 + "Qwen/Qwen1.5-7B",
11 + "Qwen/Qwen1.5-7B",
12 + tokenizer_mode="auto",
13 + trust_remote_code=False,
14 + download_dir=None,
15 + load_format="dummy",
16 + seed=0,
17 + dtype="float16",
18 + revision=None,
19 + )
20 +
21 + qwen2_model_config.hf_config.use_sliding_window = False
22 + qwen2_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
23 + assert qwen2_model_config.get_sliding_window() is None
24 +
25 + qwen2_model_config.hf_config.use_sliding_window = True
26 + assert qwen2_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
27 +
28 + mistral_model_config = ModelConfig(
29 + "mistralai/Mistral-7B-v0.1",
30 + "mistralai/Mistral-7B-v0.1",
31 + tokenizer_mode="auto",
32 + trust_remote_code=False,
33 + download_dir=None,
34 + load_format="dummy",
35 + seed=0,
36 + dtype="float16",
37 + revision=None,
38 + )
39 + mistral_model_config.hf_config.sliding_window = None
40 + assert mistral_model_config.get_sliding_window() is None
41 +
42 + mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
43 + assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
\ No newline at end of file
vllm/config.py
+12 -2
@@ -103,6 +103,7 @@ class ModelConfig:
103 # download model from ModelScope hub,
104 # lazy import so that modelscope is not required for normal use.
105 from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=C
106 +
107 if not os.path.exists(model):
108 model_path = snapshot_download(model_id=model,
109 cache_dir=download_dir,
@@ -139,7 +140,7 @@ class ModelConfig:
140 if (f not in rocm_not_supported_load_format)
141 ]
142 raise ValueError(
142 - f"load format \'{load_format}\' is not supported in ROCm. "
143 + f"load format '{load_format}' is not supported in ROCm. "
144 f"Supported load format are "
145 f"{rocm_supported_load_format}")
146
@@ -232,6 +233,15 @@ class ModelConfig:
233 f"({pipeline_parallel_size}).")
234
235 def get_sliding_window(self) -> Optional[int]:
236 + """Get the sliding window size, or None if disabled.
237 + """
238 +
239 + # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
240 + # addition to sliding window size. We check if that field is present
241 + # and if it's False, return None.
242 + if (hasattr(self.hf_config, "use_sliding_window")
243 + and not self.hf_config.use_sliding_window):
244 + return None
245 return getattr(self.hf_config, "sliding_window", None)
246
247 def get_vocab_size(self) -> int:
@@ -624,7 +634,7 @@ def _get_and_verify_dtype(
634 k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
635 if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
636 ]
627 - raise ValueError(f"dtype \'{dtype}\' is not supported in ROCm. "
637 + raise ValueError(f"dtype '{dtype}' is not supported in ROCm. "
638 f"Supported dtypes are {rocm_supported_dtypes}")
639
640 # Verify the dtype.