A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] model_max_length should consider max_model_len in tokenizer_config (#19201)
wang.yuqi committed
Jun 8, 2025 at 22:17 UTC
2ffb9b6e079b73eb10a15bb795fa886b3587f85a
2 files changed
+30
-1
vllm/config.py
+14
-1
@@ -44,7 +44,8 @@ from vllm.transformers_utils.config import (
44
ConfigFormat, get_config, get_hf_image_processor_config,
45
get_hf_text_config, get_pooling_config,
46
get_sentence_transformer_tokenizer_config, is_encoder_decoder,
47
- try_get_generation_config, try_get_safetensors_metadata, uses_mrope)
47
+ try_get_generation_config, try_get_safetensors_metadata,
48
+ try_get_tokenizer_config, uses_mrope)
49
from vllm.transformers_utils.s3_utils import S3Model
50
from vllm.transformers_utils.utils import is_s3, maybe_model_redirect
51
from vllm.utils import (DEFAULT_MAX_NUM_BATCHED_TOKENS,
@@ -1427,6 +1428,18 @@ class ModelConfig:
1428
sliding_window_len=self.get_hf_config_sliding_window(),
1429
spec_target_max_model_len=self.spec_target_max_model_len,
1430
encoder_config=self.encoder_config)
1431
+
1432
+ tokenizer_config = try_get_tokenizer_config(
1433
+ self.tokenizer,
1434
+ trust_remote_code=self.trust_remote_code,
1435
+ revision=self.tokenizer_revision)
1436
+
1437
+ if tokenizer_config is None:
1438
+ return max_model_len
1439
+
1440
+ model_max_length = tokenizer_config.get("model_max_length",
1441
+ max_model_len)
1442
+ max_model_len = min(max_model_len, model_max_length)
1443
return max_model_len
1444
1445
vllm/transformers_utils/config.py
+16
@@ -23,6 +23,7 @@ from transformers.models.auto.image_processing_auto import (
23
get_image_processor_config)
24
from transformers.models.auto.modeling_auto import (
25
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
26
+from transformers.models.auto.tokenization_auto import get_tokenizer_config
27
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
28
29
from vllm import envs
@@ -867,3 +868,18 @@ def try_get_safetensors_metadata(
868
"Error retrieving safetensors")
869
except Exception:
870
return None
871
+
872
+
873
+def try_get_tokenizer_config(
874
+ pretrained_model_name_or_path: Union[str, os.PathLike],
875
+ trust_remote_code: bool,
876
+ revision: Optional[str] = None,
877
+) -> Optional[dict[str, Any]]:
878
+ try:
879
+ return get_tokenizer_config(
880
+ pretrained_model_name_or_path,
881
+ trust_remote_code=trust_remote_code,
882
+ revision=revision,
883
+ )
884
+ except Exception:
885
+ return None