@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4abed65c58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[VLM] Disallow overflowing `max_model_len` for multimodal models (#7998)

Cyrus Leung committed Aug 30, 2024 at 08:49 UTC 4abed65c5806d0514432d102f959a1c84d341171
2 files changed +35 -3
tests/models/test_llava.py
+17
@@ -179,3 +179,20 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
179 num_logprobs=num_logprobs,
180 tensor_parallel_size=1,
181 )
182 +
183 +
184 +@pytest.mark.parametrize("model", models)
185 +def test_context_length_too_short(vllm_runner, image_assets, model):
186 + images = [asset.pil_image for asset in image_assets]
187 +
188 + with pytest.raises(ValueError, match="too long to fit into the model"):
189 + vllm_model = vllm_runner(
190 + model,
191 + max_model_len=128, # LLaVA has a feature size of 576
192 + enforce_eager=True,
193 + )
194 +
195 + with vllm_model:
196 + vllm_model.generate_greedy([HF_IMAGE_PROMPTS[0]],
197 + max_tokens=1,
198 + images=[images[0]])
vllm/engine/llm_engine.py
+18 -3
@@ -2010,7 +2010,22 @@ class LLMEngine:
2010
2011 def _validate_model_inputs(self, inputs: Union[LLMInputs,
2012 EncoderDecoderLLMInputs]):
2013 - prompt_key = "encoder_prompt_token_ids" \
2014 - if self.is_encoder_decoder_model() else "prompt_token_ids"
2015 - if not inputs.get(prompt_key):
2013 + if self.is_encoder_decoder_model():
2014 + prompt_ids = inputs.get("encoder_prompt_token_ids")
2015 + else:
2016 + prompt_ids = inputs.get("prompt_token_ids")
2017 +
2018 + if prompt_ids is None or len(prompt_ids) == 0:
2019 raise ValueError("Prompt cannot be empty")
2020 +
2021 + if self.model_config.multimodal_config is not None:
2022 + max_prompt_len = self.model_config.max_model_len
2023 +
2024 + if len(prompt_ids) > max_prompt_len:
2025 + raise ValueError(
2026 + f"The prompt (total length {len(prompt_ids)}) is too long "
2027 + f"to fit into the model (context length {max_prompt_len}). "
2028 + "Make sure that `max_model_len` is no smaller than the "
2029 + "number of text tokens plus multimodal tokens. For image "
2030 + "inputs, the number of image tokens depends on the number "
2031 + "of images, and possibly their aspect ratios as well.")