A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Command-R Max Model Length (#3727)
Roger Wang committed
Mar 29, 2024 at 12:27 UTC
97356f3c7e2fcaf1f8e17300eaf0b20b35eccb9d
1 file changed
+22
-9
vllm/config.py
+22
-9
@@ -765,15 +765,20 @@ def _get_and_verify_max_len(
765
"max_seq_len",
766
# ChatGLM2
767
"seq_length",
768
+ # Command-R
769
+ "model_max_length",
770
# Others
771
"max_sequence_length",
772
"max_seq_length",
773
"seq_len",
774
]
775
+ max_len_key = None
776
for key in possible_keys:
774
- max_len_key = getattr(hf_config, key, None)
775
- if max_len_key is not None:
776
- derived_max_model_len = min(derived_max_model_len, max_len_key)
777
+ max_len = getattr(hf_config, key, None)
778
+ if max_len is not None:
779
+ max_len_key = key if max_len < derived_max_model_len \
780
+ else max_len_key
781
+ derived_max_model_len = min(derived_max_model_len, max_len)
782
if derived_max_model_len == float("inf"):
783
if max_model_len is not None:
784
# If max_model_len is specified, we use it.
@@ -799,10 +804,18 @@ def _get_and_verify_max_len(
804
if max_model_len is None:
805
max_model_len = derived_max_model_len
806
elif max_model_len > derived_max_model_len:
802
- raise ValueError(
803
- f"User-specified max_model_len ({max_model_len}) is greater than "
804
- f"the derived max_model_len ({max_len_key}={derived_max_model_len}"
805
- " in model's config.json). This may lead to incorrect model "
806
- "outputs or CUDA errors. Make sure the value is correct and "
807
- "within the model context size.")
807
+ # Some models might have a separate key for specifying model_max_length
808
+ # that will be bigger than derived_max_model_len. We compare user input
809
+ # with model_max_length and allow this override when it's smaller.
810
+ model_max_length = getattr(hf_config, "model_max_length", None)
811
+ if model_max_length is not None and max_model_len <= model_max_length:
812
+ pass
813
+ else:
814
+ raise ValueError(
815
+ f"User-specified max_model_len ({max_model_len}) is greater "
816
+ "than the derived max_model_len "
817
+ f"({max_len_key}={derived_max_model_len} or model_max_length="
818
+ f"{model_max_length} in model's config.json). This may lead "
819
+ "to incorrect model outputs or CUDA errors. Make sure the "
820
+ "value is correct and within the model context size.")
821
return int(max_model_len)