A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[VLM] Disallow overflowing `max_model_len` for multimodal models (#7998)
Cyrus Leung committed
Aug 30, 2024 at 08:49 UTC
4abed65c5806d0514432d102f959a1c84d341171
2 files changed
+35
-3
tests/models/test_llava.py
+17
@@ -179,3 +179,20 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
179
num_logprobs=num_logprobs,
180
tensor_parallel_size=1,
181
)
182
+
183
+
184
+@pytest.mark.parametrize("model", models)
185
+def test_context_length_too_short(vllm_runner, image_assets, model):
186
+ images = [asset.pil_image for asset in image_assets]
187
+
188
+ with pytest.raises(ValueError, match="too long to fit into the model"):
189
+ vllm_model = vllm_runner(
190
+ model,
191
+ max_model_len=128, # LLaVA has a feature size of 576
192
+ enforce_eager=True,
193
+ )
194
+
195
+ with vllm_model:
196
+ vllm_model.generate_greedy([HF_IMAGE_PROMPTS[0]],
197
+ max_tokens=1,
198
+ images=[images[0]])
vllm/engine/llm_engine.py
+18
-3
@@ -2010,7 +2010,22 @@ class LLMEngine:
2010
2011
def _validate_model_inputs(self, inputs: Union[LLMInputs,
2012
EncoderDecoderLLMInputs]):
2013
- prompt_key = "encoder_prompt_token_ids" \
2014
- if self.is_encoder_decoder_model() else "prompt_token_ids"
2015
- if not inputs.get(prompt_key):
2013
+ if self.is_encoder_decoder_model():
2014
+ prompt_ids = inputs.get("encoder_prompt_token_ids")
2015
+ else:
2016
+ prompt_ids = inputs.get("prompt_token_ids")
2017
+
2018
+ if prompt_ids is None or len(prompt_ids) == 0:
2019
raise ValueError("Prompt cannot be empty")
2020
+
2021
+ if self.model_config.multimodal_config is not None:
2022
+ max_prompt_len = self.model_config.max_model_len
2023
+
2024
+ if len(prompt_ids) > max_prompt_len:
2025
+ raise ValueError(
2026
+ f"The prompt (total length {len(prompt_ids)}) is too long "
2027
+ f"to fit into the model (context length {max_prompt_len}). "
2028
+ "Make sure that `max_model_len` is no smaller than the "
2029
+ "number of text tokens plus multimodal tokens. For image "
2030
+ "inputs, the number of image tokens depends on the number "
2031
+ "of images, and possibly their aspect ratios as well.")