@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ebc0b9640

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

[Bugfix] Proper input validation for multi-modal encoder-decoder models (#16156)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 9, 2025 at 00:45 UTC 4ebc0b96401ab908e72f894138de154efbfdffd6

10 files changed +113 -62

examples/offline_inference/encoder_decoder_multimodal.py

+1 -1

@@ -56,7 +56,7 @@ def run_florence2():

56 def run_mllama():

57 engine_args = EngineArgs(

58 model="meta-llama/Llama-3.2-11B-Vision-Instruct",

59 - max_model_len=4096,

59 + max_model_len=8192,

60 max_num_seqs=2,

61 limit_mm_per_prompt={"image": 1},

62 dtype="half",

examples/offline_inference/vision_language.py

+1 -1

@@ -556,7 +556,7 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:

556 # The configuration below has been confirmed to launch on a single L40 GPU.

557 engine_args = EngineArgs(

558 model=model_name,

559 - max_model_len=4096,

559 + max_model_len=8192,

560 max_num_seqs=2,

561 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

562 )

examples/offline_inference/vision_language_multi_image.py

+2 -2

@@ -318,8 +318,8 @@ def load_mllama(question: str, image_urls: list[str]) -> ModelRequestData:

318 # The configuration below has been confirmed to launch on a single L40 GPU.

319 engine_args = EngineArgs(

320 model=model_name,

321 - max_model_len=4096,

322 - max_num_seqs=16,

321 + max_model_len=8192,

322 + max_num_seqs=2,

323 limit_mm_per_prompt={"image": len(image_urls)},

324 )

325

tests/engine/test_short_mm_context.py

+2 -1

@@ -18,7 +18,8 @@ models = ["llava-hf/llava-1.5-7b-hf"]

18 def test_context_length_too_short(vllm_runner, image_assets, model):

19 images = [asset.pil_image for asset in image_assets]

21 - with pytest.raises(ValueError, match="too long to fit into the model"):

21 + with pytest.raises(ValueError,

22 + match="longer than the maximum model length"):

23 vllm_model = vllm_runner(

24 model,

25 max_model_len=128, # LLaVA has a feature size of 576

tests/entrypoints/llm/test_prompt_validation.py

+1 -1

@@ -15,7 +15,7 @@ def v1(run_with_both_engines):

16 def test_empty_prompt():

17 llm = LLM(model="openai-community/gpt2", enforce_eager=True)

18 - with pytest.raises(ValueError, match='Prompt cannot be empty'):

18 + with pytest.raises(ValueError, match='decoder prompt cannot be empty'):

19 llm.generate([""])

tests/entrypoints/openai/test_prompt_validation.py

+1 -1

@@ -17,7 +17,7 @@ async def test_empty_prompt():

17 client = remote_server.get_async_client()

19 with pytest.raises(openai.BadRequestError,

20 - match=re.compile('.+Prompt cannot be empty.+')):

20 + match="decoder prompt cannot be empty"):

21 await client.completions.create(model=model_name,

22 prompt="",

23 max_tokens=5,

tests/models/encoder_decoder/vision_language/test_mllama.py

+4 -4

@@ -211,7 +211,7 @@ def _run_test(

211 # max_model_len should be greater than image_feature_size

212 with vllm_runner(model,

213 dtype=dtype,

214 - max_model_len=4096,

214 + max_model_len=8192,

215 max_num_seqs=3,

216 tensor_parallel_size=tensor_parallel_size,

217 distributed_executor_backend=distributed_executor_backend,

@@ -422,7 +422,7 @@ def test_bnb_regression(

422 llm = LLM(

423 model=model,

424 dtype=dtype,

425 - max_model_len=4096,

425 + max_model_len=8192,

426 max_num_seqs=2,

427 quantization="bitsandbytes",

428 )

@@ -475,7 +475,7 @@ def test_explicit_implicit_prompt(

475 llm = LLM(

476 model=model,

477 dtype=dtype,

478 - max_model_len=4096,

478 + max_model_len=8192,

479 max_num_seqs=2,

480 tensor_parallel_size=1,

481 )

@@ -506,7 +506,7 @@ def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,

506 with global_force_attn_backend_context_manager(attn_backend), vllm_runner(

507 model,

508 dtype=dtype,

509 - max_model_len=4096,

509 + max_model_len=8192,

510 max_num_seqs=2,

511 tensor_parallel_size=1,

512 limit_mm_per_prompt={"image":

vllm/engine/llm_engine.py

+46 -17

@@ -8,7 +8,7 @@ from contextlib import contextmanager

8 from dataclasses import dataclass

9 from functools import partial

10 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,

11 - Iterable, List, Mapping, NamedTuple, Optional)

11 + Iterable, List, Literal, Mapping, NamedTuple, Optional)

12 from typing import Sequence as GenericSequence

13 from typing import Set, Type, Union, cast, overload

@@ -30,7 +30,7 @@ from vllm.entrypoints.openai.logits_processors import (

30 get_logits_processors as get_openai_logits_processors)

31 from vllm.executor.executor_base import ExecutorBase

32 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,

33 - PromptType)

33 + PromptType, SingletonInputs)

34 from vllm.inputs.parse import is_token_prompt, split_enc_dec_inputs

35 from vllm.inputs.preprocess import InputPreprocessor

36 from vllm.logger import init_logger

@@ -40,6 +40,7 @@ from vllm.model_executor.guided_decoding import (

40 get_local_guided_decoding_logits_processor)

41 from vllm.model_executor.layers.sampler import SamplerOutput

42 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

43 +from vllm.multimodal.processing import EncDecMultiModalProcessor

44 from vllm.outputs import (PoolingRequestOutput, RequestOutput,

45 RequestOutputFactory)

46 from vllm.pooling_params import PoolingParams

@@ -2029,29 +2030,57 @@ class LLMEngine:

2030 lora_request: Optional[LoRARequest]):

2031 encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)

2032

2032 - # For encoder-decoder multimodal models, the max_prompt_len

2033 - # restricts the decoder prompt length

2034 - if self.model_config.is_multimodal_model:

2035 - prompt_inputs = decoder_inputs

2036 - else:

2037 - prompt_inputs = encoder_inputs or decoder_inputs

2033 + if encoder_inputs is not None:

2034 + self._validate_model_input(encoder_inputs,

2035 + lora_request,

2036 + prompt_type="encoder")

2037

2039 - prompt_ids = prompt_inputs["prompt_token_ids"]

2038 + self._validate_model_input(decoder_inputs,

2039 + lora_request,

2040 + prompt_type="decoder")

2041

2041 - if prompt_ids is None or len(prompt_ids) == 0:

2042 - raise ValueError("Prompt cannot be empty")

2042 + def _validate_model_input(

2043 + self,

2044 + prompt_inputs: SingletonInputs,

2045 + lora_request: Optional[LoRARequest],

2046 + *,

2047 + prompt_type: Literal["encoder", "decoder"],

2048 + ):

2049 + if prompt_type == "encoder" and self.tokenizer is not None:

2050 + tokenizer = self.tokenizer.get_lora_tokenizer(lora_request)

2051 + model_config = self.model_config

2052

2044 - if self.model_config.is_multimodal_model:

2045 - max_prompt_len = self.model_config.max_model_len

2053 + if model_config.is_multimodal_model:

2054 + mm_registry = self.input_preprocessor.mm_registry

2055 + mm_processor = mm_registry.create_processor(

2056 + model_config, tokenizer=tokenizer)

2057 + assert isinstance(mm_processor, EncDecMultiModalProcessor)

2058

2047 - if len(prompt_ids) > max_prompt_len:

2048 - raise ValueError(

2049 - f"The prompt (total length {len(prompt_ids)}) is too long "

2050 - f"to fit into the model (context length {max_prompt_len}). "

2059 + if mm_processor.pad_dummy_encoder_prompt:

2060 + return # Skip encoder length check for Whisper

2061 +

2062 + prompt_ids = prompt_inputs["prompt_token_ids"]

2063 +

2064 + if not prompt_ids:

2065 + raise ValueError(f"The {prompt_type} prompt cannot be empty")

2066 +

2067 + max_prompt_len = self.model_config.max_model_len

2068 + if len(prompt_ids) >= max_prompt_len:

2069 + if self.model_config.is_multimodal_model:

2070 + suggestion = (

2071 "Make sure that `max_model_len` is no smaller than the "

2072 "number of text tokens plus multimodal tokens. For image "

2073 "inputs, the number of image tokens depends on the number "

2074 "of images, and possibly their aspect ratios as well.")

2075 + else:

2076 + suggestion = (

2077 + "Make sure that `max_model_len` is no smaller than the "

2078 + "number of text tokens.")

2079 +

2080 + raise ValueError(

2081 + f"The {prompt_type} prompt (length {len(prompt_ids)}) is "

2082 + f"longer than the maximum model length of {max_prompt_len}. "

2083 + f"{suggestion}")

2084

2085 # TODO: Find out how many placeholder tokens are there so we can

2086 # check that chunked prefill does not truncate them

vllm/multimodal/profiling.py

+6 -7

@@ -213,8 +213,12 @@ class MultiModalProfiler(Generic[_I]):

213

214 total_len = len(encoder_prompt_token_ids)

215

216 - # Encoder-decoder multimodal models only support v0

217 - if total_len > seq_len:

216 + processor = cast(EncDecMultiModalProcessor, self.processor)

217 + if processor.pad_dummy_encoder_prompt:

218 + num_tokens_to_pad = max(total_len, seq_len) - total_len

219 + encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)

220 + # NOTE: Whisper allows total_len > seq_len.

221 + elif total_len > seq_len and not envs.VLLM_USE_V1:

222 # `max_num_batched_tokens` is defined by `SchedulerConfig`

223 logger.warning_once(

224 "The encoder sequence length used for profiling ("

@@ -229,11 +233,6 @@ class MultiModalProfiler(Generic[_I]):

233 "increase `max_model_len`, reduce `max_num_seqs`, "

234 "and/or reduce `mm_counts`.")

235

232 - processor = cast(EncDecMultiModalProcessor, self.processor)

233 - if processor.pad_dummy_encoder_prompt:

234 - num_tokens_to_pad = max(total_len, seq_len) - total_len

235 - encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)

236 -

236 return DummyEncoderData(encoder_prompt_token_ids)

237

238 def get_decoder_dummy_data(

vllm/v1/engine/processor.py

+49 -27

@@ -2,16 +2,17 @@

3 import time

4 from collections.abc import Mapping

5 -from typing import Optional, Union

5 +from typing import Literal, Optional, Union

7 from vllm.config import VllmConfig

8 -from vllm.inputs import ProcessorInputs, PromptType

8 +from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs

9 from vllm.inputs.parse import split_enc_dec_inputs

10 from vllm.inputs.preprocess import InputPreprocessor

11 from vllm.lora.request import LoRARequest

12 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,

13 MultiModalRegistry)

14 from vllm.multimodal.inputs import PlaceholderRange

15 +from vllm.multimodal.processing import EncDecMultiModalProcessor

16 from vllm.multimodal.utils import merge_and_sort_multimodal_metadata

17 from vllm.pooling_params import PoolingParams

18 from vllm.prompt_adapter.request import PromptAdapterRequest

@@ -287,41 +288,62 @@ class Processor:

288 lora_request: Optional[LoRARequest] = None):

289 encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)

290

290 - # For encoder-decoder multimodal models, the max_prompt_len

291 - # restricts the decoder prompt length

292 - if self.model_config.is_multimodal_model:

293 - prompt_inputs = decoder_inputs

294 - else:

295 - prompt_inputs = encoder_inputs or decoder_inputs

291 + if encoder_inputs is not None:

292 + self._validate_model_input(encoder_inputs,

293 + lora_request,

294 + prompt_type="encoder")

295

297 - prompt_ids = prompt_inputs["prompt_token_ids"]

296 + self._validate_model_input(decoder_inputs,

297 + lora_request,

298 + prompt_type="decoder")

299

299 - if prompt_ids is None or len(prompt_ids) == 0:

300 - raise ValueError("Prompt cannot be empty")

300 + def _validate_model_input(

301 + self,

302 + prompt_inputs: SingletonInputs,

303 + lora_request: Optional[LoRARequest],

304 + *,

305 + prompt_type: Literal["encoder", "decoder"],

306 + ):

307 + tokenizer = self.tokenizer.get_lora_tokenizer(lora_request)

308

302 - max_input_id = max(prompt_ids)

303 - max_allowed = self.tokenizer.get_lora_tokenizer(

304 - lora_request).max_token_id

305 - if max_input_id > max_allowed:

306 - raise ValueError(

307 - "Token id {} is out of vocabulary".format(max_input_id))

309 + if prompt_type == "encoder":

310 + model_config = self.model_config

311

309 - if len(prompt_ids) >= self.model_config.max_model_len:

310 - raise ValueError(

311 - f"Prompt length of {len(prompt_ids)} is longer than the "

312 - f"maximum model length of {self.model_config.max_model_len}.")

312 + if model_config.is_multimodal_model:

313 + mm_registry = self.input_preprocessor.mm_registry

314 + mm_processor = mm_registry.create_processor(

315 + model_config, tokenizer=tokenizer)

316 + assert isinstance(mm_processor, EncDecMultiModalProcessor)

317

314 - if self.model_config.is_multimodal_model:

315 - max_prompt_len = self.model_config.max_model_len

318 + if mm_processor.pad_dummy_encoder_prompt:

319 + return # Skip encoder length check for Whisper

320

317 - if len(prompt_ids) > max_prompt_len:

318 - raise ValueError(

319 - f"The prompt (total length {len(prompt_ids)}) is too long "

320 - f"to fit into the model (context length {max_prompt_len}). "

321 + prompt_ids = prompt_inputs["prompt_token_ids"]

322 +

323 + if not prompt_ids:

324 + raise ValueError(f"The {prompt_type} prompt cannot be empty")

325 +

326 + max_input_id = max(prompt_ids)

327 + if max_input_id > tokenizer.max_token_id:

328 + raise ValueError(f"Token id {max_input_id} is out of vocabulary")

329 +

330 + max_prompt_len = self.model_config.max_model_len

331 + if len(prompt_ids) >= max_prompt_len:

332 + if self.model_config.is_multimodal_model:

333 + suggestion = (

334 "Make sure that `max_model_len` is no smaller than the "

335 "number of text tokens plus multimodal tokens. For image "

336 "inputs, the number of image tokens depends on the number "

337 "of images, and possibly their aspect ratios as well.")

338 + else:

339 + suggestion = (

340 + "Make sure that `max_model_len` is no smaller than the "

341 + "number of text tokens.")

342 +

343 + raise ValueError(

344 + f"The {prompt_type} prompt (length {len(prompt_ids)}) is "

345 + f"longer than the maximum model length of {max_prompt_len}. "

346 + f"{suggestion}")

347

348 # TODO: Find out how many placeholder tokens are there so we can

349 # check that chunked prefill does not truncate them