@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97356f3c7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Command-R Max Model Length (#3727)

Roger Wang committed Mar 29, 2024 at 12:27 UTC 97356f3c7e2fcaf1f8e17300eaf0b20b35eccb9d
1 file changed +22 -9
vllm/config.py
+22 -9
@@ -765,15 +765,20 @@ def _get_and_verify_max_len(
765 "max_seq_len",
766 # ChatGLM2
767 "seq_length",
768 + # Command-R
769 + "model_max_length",
770 # Others
771 "max_sequence_length",
772 "max_seq_length",
773 "seq_len",
774 ]
775 + max_len_key = None
776 for key in possible_keys:
774 - max_len_key = getattr(hf_config, key, None)
775 - if max_len_key is not None:
776 - derived_max_model_len = min(derived_max_model_len, max_len_key)
777 + max_len = getattr(hf_config, key, None)
778 + if max_len is not None:
779 + max_len_key = key if max_len < derived_max_model_len \
780 + else max_len_key
781 + derived_max_model_len = min(derived_max_model_len, max_len)
782 if derived_max_model_len == float("inf"):
783 if max_model_len is not None:
784 # If max_model_len is specified, we use it.
@@ -799,10 +804,18 @@ def _get_and_verify_max_len(
804 if max_model_len is None:
805 max_model_len = derived_max_model_len
806 elif max_model_len > derived_max_model_len:
802 - raise ValueError(
803 - f"User-specified max_model_len ({max_model_len}) is greater than "
804 - f"the derived max_model_len ({max_len_key}={derived_max_model_len}"
805 - " in model's config.json). This may lead to incorrect model "
806 - "outputs or CUDA errors. Make sure the value is correct and "
807 - "within the model context size.")
807 + # Some models might have a separate key for specifying model_max_length
808 + # that will be bigger than derived_max_model_len. We compare user input
809 + # with model_max_length and allow this override when it's smaller.
810 + model_max_length = getattr(hf_config, "model_max_length", None)
811 + if model_max_length is not None and max_model_len <= model_max_length:
812 + pass
813 + else:
814 + raise ValueError(
815 + f"User-specified max_model_len ({max_model_len}) is greater "
816 + "than the derived max_model_len "
817 + f"({max_len_key}={derived_max_model_len} or model_max_length="
818 + f"{model_max_length} in model's config.json). This may lead "
819 + "to incorrect model outputs or CUDA errors. Make sure the "
820 + "value is correct and within the model context size.")
821 return int(max_model_len)