A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix] Fix the max_seq_len limit of 16384 for DeepSeek models (#20322)
Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com>
WangHuaqiang committed
Jul 2, 2025 at 20:53 UTC
ccbfb1d1c96352e9022093c6d362423f1fc56918
2 files changed
+13
-4
tests/test_config.py
+2
@@ -412,6 +412,8 @@ def test_load_config_pt_load_map_location(pt_load_map_location):
412
("BAAI/bge-reranker-base", None, 512, False),
413
("BAAI/bge-reranker-base", 256, 256, False),
414
("BAAI/bge-reranker-base", 513, 512, True),
415
+ ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", None, 131072, False),
416
+ ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True),
417
])
418
def test_get_and_verify_max_len(model_id, max_model_len, expected_max_len,
419
should_raise):
vllm/config.py
+11
-4
@@ -1442,10 +1442,17 @@ class ModelConfig:
1442
return getattr(self.hf_config, "matryoshka_dimensions", None)
1443
1444
def get_and_verify_max_len(self, max_model_len: int):
1445
- tokenizer_config = try_get_tokenizer_config(
1446
- self.tokenizer,
1447
- trust_remote_code=self.trust_remote_code,
1448
- revision=self.tokenizer_revision)
1445
+ # For pooling models, the tokenizer's `model_max_length` is often a
1446
+ # reliable source for the maximum sequence length. However, for
1447
+ # generative models, this can be incorrect and unduly limit the
1448
+ # context window (e.g., DeepSeek-R1). Therefore, we only consider
1449
+ # tokenizer_config for pooling models.
1450
+ tokenizer_config = None
1451
+ if self.runner_type == "pooling":
1452
+ tokenizer_config = try_get_tokenizer_config(
1453
+ self.tokenizer,
1454
+ trust_remote_code=self.trust_remote_code,
1455
+ revision=self.tokenizer_revision)
1456
max_model_len = _get_and_verify_max_len(
1457
hf_config=self.hf_text_config,
1458
tokenizer_config=tokenizer_config,