@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b73a2f498
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Spec Decoding] Use target model max length as default for draft model (#7706)

Nick Hill committed Aug 21, 2024 at 12:23 UTC 9b73a2f498e8bf8e18305a8afea84536e9330088
1 file changed +10 -1
vllm/config.py
+10 -1
@@ -127,6 +127,7 @@ class ModelConfig:
127 rope_theta: Optional[float] = None,
128 tokenizer_revision: Optional[str] = None,
129 max_model_len: Optional[int] = None,
130 + spec_target_max_model_len: Optional[int] = None,
131 quantization: Optional[str] = None,
132 quantization_param_path: Optional[str] = None,
133 enforce_eager: Optional[bool] = None,
@@ -210,7 +211,8 @@ class ModelConfig:
211 hf_config=self.hf_text_config,
212 max_model_len=max_model_len,
213 disable_sliding_window=self.disable_sliding_window,
213 - sliding_window_len=self.get_hf_config_sliding_window())
214 + sliding_window_len=self.get_hf_config_sliding_window(),
215 + spec_target_max_model_len=spec_target_max_model_len)
216 self.served_model_name = get_served_model_name(model,
217 served_model_name)
218 self.multimodal_config = self._init_multimodal_config(
@@ -1134,6 +1136,7 @@ class SpeculativeConfig:
1136 code_revision=draft_code_revision,
1137 tokenizer_revision=target_model_config.tokenizer_revision,
1138 max_model_len=None,
1139 + spec_target_max_model_len=target_model_config.max_model_len,
1140 quantization=draft_quantization,
1141 enforce_eager=target_model_config.enforce_eager,
1142 max_seq_len_to_capture=target_model_config.
@@ -1563,6 +1566,7 @@ def _get_and_verify_max_len(
1566 max_model_len: Optional[int],
1567 disable_sliding_window: bool,
1568 sliding_window_len: Optional[int],
1569 + spec_target_max_model_len: Optional[int] = None,
1570 ) -> int:
1571 """Get and verify the model's maximum length."""
1572 derived_max_model_len = float("inf")
@@ -1605,6 +1609,11 @@ def _get_and_verify_max_len(
1609 # If max_model_len is specified, we use it.
1610 return max_model_len
1611
1612 + if spec_target_max_model_len is not None:
1613 + # If this is a speculative draft model, we use the max model len
1614 + # from the target model.
1615 + return spec_target_max_model_len
1616 +
1617 default_max_len = 2048
1618 logger.warning(
1619 "The model's config.json does not contain any of the following "