@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ad9951c41
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Input] Remove unused `prompt` field (#26097)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Oct 3, 2025 at 15:23 UTC 0ad9951c416d33c5da4f7a504fb162cbe62386f5
15 files changed +67 -101
tests/models/multimodal/processing/test_transformers.py
+2 -1
@@ -37,4 +37,5 @@ def test_multimodal_processor(model_id):
37 hf_processor_mm_kwargs={},
38 )
39
40 - assert str_processed_inputs["prompt"] == ids_processed_inputs["prompt"]
40 + assert (str_processed_inputs["prompt_token_ids"]
41 + == ids_processed_inputs["prompt_token_ids"])
vllm/engine/protocol.py
+8 -3
@@ -94,10 +94,15 @@ class EngineClient(ABC):
94 # this happens again in generation, so the double expansion causes
95 # a mismatch.
96 # TODO - would be ideal to handle this more gracefully.
97 - prompt_token_ids = prompt.get("prompt_token_ids")
98 - multi_modal_data = prompt.get("multi_modal_data")
97 + if isinstance(prompt, str):
98 + prompt_text = prompt
99 + prompt_token_ids = []
100 + multi_modal_data = None
101 + else:
102 + prompt_text = prompt.get("prompt")
103 + prompt_token_ids = prompt.get("prompt_token_ids", [])
104 + multi_modal_data = prompt.get("multi_modal_data")
105
100 - prompt_text = processed_inputs.get("prompt")
106 mm_processor_kwargs = processed_inputs.get("mm_processor_kwargs")
107
108 tokenized_length = len(prompt_token_ids)
vllm/inputs/data.py
-8
@@ -205,11 +205,6 @@ class TokenInputs(TypedDict):
205 prompt_token_ids: list[int]
206 """The token IDs of the prompt."""
207
208 - prompt: NotRequired[str]
209 - """
210 - The original prompt text corresponding to the token IDs, if available.
211 - """
212 -
208 cache_salt: NotRequired[str]
209 """
210 Optional cache salt to be used for prefix caching.
@@ -218,15 +213,12 @@ class TokenInputs(TypedDict):
213
214 def token_inputs(
215 prompt_token_ids: list[int],
221 - prompt: Optional[str] = None,
216 cache_salt: Optional[str] = None,
217 ) -> TokenInputs:
218 """Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional
219 values."""
220 inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
221
228 - if prompt is not None:
229 - inputs["prompt"] = prompt
222 if cache_salt is not None:
223 inputs["cache_salt"] = cache_salt
224
vllm/inputs/preprocess.py
+17 -20
@@ -16,9 +16,10 @@ from vllm.multimodal.processing import BaseMultiModalProcessor
16 from vllm.transformers_utils.tokenizer import AnyTokenizer
17
18 from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
19 - EncoderDecoderInputs, ProcessorInputs, PromptType,
20 - SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
21 - TokensPrompt, embeds_inputs, token_inputs)
19 + EncoderDecoderInputs, ExplicitEncoderDecoderPrompt,
20 + ProcessorInputs, PromptType, SingletonInputs,
21 + SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
22 + embeds_inputs, token_inputs)
23 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
24
25 logger = init_logger(__name__)
@@ -322,7 +323,7 @@ class InputPreprocessor:
323 mm_uuids=mm_uuids,
324 )
325 else:
325 - inputs = token_inputs(prompt_token_ids=prompt_token_ids)
326 + inputs = token_inputs(prompt_token_ids)
327
328 if cache_salt := parsed_content.get("cache_salt"):
329 inputs["cache_salt"] = cache_salt
@@ -352,10 +353,7 @@ class InputPreprocessor:
353 prompt_text,
354 tokenization_kwargs=tokenization_kwargs,
355 )
355 - inputs = token_inputs(
356 - prompt=prompt_text,
357 - prompt_token_ids=prompt_token_ids,
358 - )
356 + inputs = token_inputs(prompt_token_ids)
357
358 if cache_salt := parsed_content.get("cache_salt"):
359 inputs["cache_salt"] = cache_salt
@@ -473,22 +471,17 @@ class InputPreprocessor:
471 decoder_inputs: SingletonInputs
472
473 if inputs["type"] == "multimodal": # Multimodal data inputs
476 - if not ("encoder_prompt" in inputs
477 - and "encoder_prompt_token_ids" in inputs):
474 + if "encoder_prompt_token_ids" not in inputs:
475 raise RuntimeError("You should register an encoder-decoder "
476 "multi-modal processor for encoder-decoder "
477 "models.")
478 inputs = cast(MultiModalEncDecInputs, inputs)
479
483 - encoder_inputs = token_inputs(
484 - prompt=inputs["encoder_prompt"],
485 - prompt_token_ids=inputs["encoder_prompt_token_ids"],
486 - )
480 + encoder_inputs = token_inputs(inputs["encoder_prompt_token_ids"])
481
482 decoder_prompt_inputs = decoder_inputs_to_override or inputs
483 decoder_inputs = MultiModalInputs(
484 type="multimodal",
491 - prompt=decoder_prompt_inputs.get("prompt", ""),
485 prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
486 mm_kwargs=inputs["mm_kwargs"],
487 mm_hashes=inputs["mm_hashes"],
@@ -498,7 +491,7 @@ class InputPreprocessor:
491 decoder_inputs["cache_salt"] = cache_salt
492
493 elif inputs["type"] == "token": # Text-only inputs
501 - encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
494 + encoder_inputs = token_inputs(prompt_token_ids=[])
495 decoder_inputs = decoder_inputs_to_override or inputs
496 else:
497 assert_never(inputs) # type: ignore[arg-type]
@@ -549,12 +542,14 @@ class InputPreprocessor:
542 decoder_inputs: Optional[SingletonInputs]
543
544 if is_explicit_encoder_decoder_prompt(prompt):
545 + # `cast` is needed for mypy, but not pyright
546 + prompt_ = cast(ExplicitEncoderDecoderPrompt, prompt)
547 encoder_inputs = self._prompt_to_llm_inputs(
553 - prompt["encoder_prompt"],
548 + prompt_["encoder_prompt"],
549 tokenization_kwargs=tokenization_kwargs,
550 mm_uuids=mm_uuids,
551 )
557 - if (decoder_input := prompt["decoder_prompt"]) is None:
552 + if (decoder_input := prompt_["decoder_prompt"]) is None:
553 decoder_inputs = None
554 else:
555 decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
@@ -565,8 +560,9 @@ class InputPreprocessor:
560 self._split_enc_dec_mm_inputs(encoder_inputs,
561 decoder_inputs))
562 else:
563 + # `cast` is needed for mypy, but not pyright
564 inputs = self._prompt_to_llm_inputs(
569 - prompt,
565 + cast(SingletonPrompt, prompt),
566 tokenization_kwargs=tokenization_kwargs,
567 mm_uuids=mm_uuids,
568 )
@@ -641,8 +637,9 @@ class InputPreprocessor:
637 "to decoder-only models")
638
639 # Decoder-only operation
640 + # `cast` is needed for mypy, but not pyright
641 return self._process_decoder_only_prompt(
645 - prompt,
642 + cast(SingletonPrompt, prompt),
643 tokenization_kwargs=tokenization_kwargs,
644 mm_uuids=mm_uuids,
645 )
vllm/model_executor/models/llava.py
+1 -2
@@ -778,7 +778,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
778 )
779 ], mm_item_counts)
780
781 - prompt_ids, prompt, _ = self._apply_prompt_updates(
781 + prompt_ids, _ = self._apply_prompt_updates(
782 result["prompt_token_ids"],
783 mantis_mm_repls,
784 )
@@ -798,7 +798,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
798
799 return MultiModalInputs(
800 type="multimodal",
801 - prompt=prompt,
801 prompt_token_ids=prompt_ids,
802 mm_kwargs=mm_kwargs,
803 mm_hashes=mm_hashes,
vllm/model_executor/models/paligemma.py
-1
@@ -219,7 +219,6 @@ class PaliGemmaMultiModalProcessor(
219 if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:
220 prompt_token_ids.append(newline_token_id)
221 mm_inputs["prompt_token_ids"] = prompt_token_ids
222 - mm_inputs["prompt"] += newline_prompt
222
223 return mm_inputs
224
vllm/model_executor/models/phi3v.py
+5 -5
@@ -461,7 +461,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
461 self,
462 token_ids: list[int],
463 mm_prompt_updates: MultiModalPromptUpdates,
464 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
464 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
465 # align to hf behavior when there are images
466 if len(mm_prompt_updates):
467 tokenizer = self.info.get_tokenizer()
@@ -496,14 +496,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
496 for ele in sublist for e in ele
497 ]
498
499 - token_ids, text, placeholders = super()._apply_prompt_updates(
499 + token_ids, placeholders = super()._apply_prompt_updates(
500 token_ids=token_ids,
501 mm_prompt_updates=mm_prompt_updates,
502 )
503
504 # Keep the behavior in line with HF processor
505 - if text.startswith("<s> <|image|>"):
506 - text = text.replace("<s> <|image|>", "<s><|image|>", 1)
505 + if token_ids[:2] == tokenizer.encode("<s> <|image|>",
506 + add_special_tokens=False):
507 token_ids = [token_ids[0], *token_ids[2:]]
508 placeholders = {
509 modality: [
@@ -518,7 +518,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
518 for modality, ps in placeholders.items()
519 }
520
521 - return token_ids, text, placeholders
521 + return token_ids, placeholders
522
523
524 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+8 -16
@@ -63,7 +63,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
63 PromptReplacement, PromptUpdate)
64 from vllm.multimodal.profiling import BaseDummyInputsBuilder
65 from vllm.sequence import IntermediateTensors
66 -from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
66 +from vllm.transformers_utils.tokenizer import encode_tokens
67 from vllm.utils.tensor_schema import TensorSchema, TensorShape
68
69 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -316,7 +316,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
316 mm_kwargs: MultiModalKwargsItems,
317 mm_prompt_updates: MultiModalPromptUpdates,
318 is_update_applied: bool,
319 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
319 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
320 """
321 Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
322 """
@@ -341,28 +341,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
341 self._validate_mm_placeholders(
342 mm_placeholders,
343 mm_item_counts,
344 - use_audio_in_video=use_audio_in_video)
345 -
346 - tokenizer = self.info.get_tokenizer()
347 - prompt = decode_tokens(tokenizer, prompt_ids)
344 + use_audio_in_video=use_audio_in_video,
345 + )
346 else:
349 - (
350 - prompt_ids,
351 - prompt,
352 - mm_placeholders,
353 - ) = self._apply_prompt_updates(
347 + prompt_ids, mm_placeholders = self._apply_prompt_updates(
348 prompt_ids,
349 mm_prompt_updates,
350 )
351 self._validate_mm_placeholders(
352 mm_placeholders,
353 mm_item_counts,
360 - use_audio_in_video=use_audio_in_video)
361 -
362 - tokenizer = self.info.get_tokenizer()
363 - prompt = decode_tokens(tokenizer, prompt_ids)
354 + use_audio_in_video=use_audio_in_video,
355 + )
356
365 - return prompt_ids, prompt, mm_placeholders
357 + return prompt_ids, mm_placeholders
358
359 def _get_prompt_updates(
360 self,
vllm/model_executor/models/terratorch.py
-1
@@ -190,7 +190,6 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
190
191 return MultiModalInputs(
192 type="multimodal",
193 - prompt=prompt,
193 prompt_token_ids=[1],
194 mm_kwargs=mm_kwargs,
195 mm_hashes=mm_hashes,
vllm/model_executor/models/transformers.py
-1
@@ -453,7 +453,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
453
454 return MultiModalInputs(
455 type="multimodal",
456 - prompt=prompt,
456 prompt_token_ids=prompt_ids,
457 mm_kwargs=mm_kwargs,
458 mm_hashes=mm_hashes,
vllm/multimodal/inputs.py
-6
@@ -949,9 +949,6 @@ class MultiModalInputs(TypedDict):
949 type: Literal["multimodal"]
950 """The type of inputs."""
951
952 - prompt: str
953 - """The processed prompt text."""
954 -
952 prompt_token_ids: list[int]
953 """The processed token IDs which includes placeholder tokens."""
954
@@ -980,8 +977,5 @@ class MultiModalEncDecInputs(MultiModalInputs):
977 ready to be passed to vLLM internals.
978 """
979
983 - encoder_prompt: str
984 - """The processed encoder prompt text."""
985 -
980 encoder_prompt_token_ids: list[int]
981 """The processed token IDs of the encoder prompt."""
vllm/multimodal/processing.py
+7 -21
@@ -1878,7 +1878,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1878 self,
1879 token_ids: list[int],
1880 mm_prompt_updates: MultiModalPromptUpdates,
1881 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1881 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1882 tokenizer = self.info.get_tokenizer()
1883
1884 new_token_ids, match_result = self._apply_token_matches(
@@ -1896,11 +1896,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1896 # Since it is inefficient to search for all possible tokenizations
1897 # of the search text in the prompt, we instead perform string-based
1898 # updates on the decoded token IDs, then encode them back.
1899 - if all(
1899 + if not all(
1900 all(update_idx is not None for update_idx in update_idxs)
1901 for update_idxs in match_result.values()):
1902 - new_text = decode_tokens(tokenizer, new_token_ids)
1903 - else:
1902 new_text, match_result = self._apply_text_matches(
1903 decode_tokens(tokenizer, token_ids),
1904 mm_prompt_updates,
@@ -1928,7 +1926,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1926 dict(matched_updates),
1927 )
1928
1931 - return new_token_ids, new_text, placeholders
1929 + return new_token_ids, placeholders
1930
1931 def _validate_mm_kwargs(
1932 self,
@@ -1976,7 +1974,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1974 mm_kwargs: MultiModalKwargsOptionalItems,
1975 mm_prompt_updates: MultiModalPromptUpdates,
1976 is_update_applied: bool,
1979 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1977 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1978 mm_item_counts = mm_items.get_all_counts()
1979 self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1980
@@ -1986,21 +1984,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1984 mm_prompt_updates,
1985 )
1986 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1989 -
1990 - tokenizer = self.info.get_tokenizer()
1991 - prompt = decode_tokens(tokenizer, prompt_ids)
1987 else:
1993 - (
1994 - prompt_ids,
1995 - prompt,
1996 - mm_placeholders,
1997 - ) = self._apply_prompt_updates(
1988 + prompt_ids, mm_placeholders = self._apply_prompt_updates(
1989 prompt_ids,
1990 mm_prompt_updates,
1991 )
1992 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1993
2003 - return prompt_ids, prompt, mm_placeholders
1994 + return prompt_ids, mm_placeholders
1995
1996 def apply(
1997 self,
@@ -2042,7 +2033,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2033 )
2034
2035 # NOTE: tokenization_kwargs are not required to init processor
2045 - prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(
2036 + prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2037 mm_items=mm_items,
2038 prompt_ids=prompt_ids,
2039 mm_kwargs=mm_info.kwargs,
@@ -2057,7 +2048,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2048
2049 return MultiModalInputs(
2050 type="multimodal",
2060 - prompt=prompt,
2051 prompt_token_ids=prompt_ids,
2052 mm_kwargs=mm_info.kwargs,
2053 mm_hashes=mm_info.hashes,
@@ -2100,19 +2090,15 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
2090 tokenizer = self.info.get_tokenizer()
2091 decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
2092 if isinstance(decoder_prompt_raw, str):
2103 - decoder_prompt = decoder_prompt_raw
2093 decoder_prompt_ids = encode_tokens(tokenizer,
2094 decoder_prompt_raw,
2095 add_special_tokens=False)
2096 else:
2108 - decoder_prompt = decode_tokens(tokenizer, decoder_prompt_raw)
2097 decoder_prompt_ids = decoder_prompt_raw
2098
2099 mm_inputs = MultiModalEncDecInputs(
2112 - encoder_prompt=encoder_inputs["prompt"],
2100 encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
2101 **encoder_inputs)
2115 - mm_inputs["prompt"] = decoder_prompt
2102 mm_inputs["prompt_token_ids"] = decoder_prompt_ids
2103 return mm_inputs
2104
vllm/v1/engine/async_llm.py
+9 -5
@@ -281,12 +281,16 @@ class AsyncLLM(EngineClient):
281 queue = RequestOutputCollector(output_kind=params.output_kind)
282
283 # Convert Input --> Request.
284 - prompt_str, request = self.processor.process_inputs(
285 - request_id, prompt, params, arrival_time, lora_request,
286 - tokenization_kwargs, trace_headers, priority, data_parallel_rank)
284 + request = self.processor.process_inputs(request_id, prompt, params,
285 + arrival_time, lora_request,
286 + tokenization_kwargs,
287 + trace_headers, priority,
288 + data_parallel_rank)
289 + prompt_text = prompt if isinstance(prompt,
290 + str) else prompt.get("prompt")
291
292 if is_pooling or params.n == 1:
289 - await self._add_request(request, prompt_str, None, 0, queue)
293 + await self._add_request(request, prompt_text, None, 0, queue)
294 return queue
295
296 # Get the updated SamplingParams from the request, which
@@ -302,7 +306,7 @@ class AsyncLLM(EngineClient):
306 request)
307 child_request.request_id = request_id
308 child_request.sampling_params = child_params
305 - await self._add_request(child_request, prompt_str, parent_request,
309 + await self._add_request(child_request, prompt_text, parent_request,
310 idx, queue)
311 return queue
312
vllm/v1/engine/llm_engine.py
+8 -5
@@ -227,15 +227,18 @@ class LLMEngine:
227 f"request_id must be a string, got {type(request_id)}")
228
229 # Process raw inputs into the request.
230 - prompt_str, request = self.processor.process_inputs(
231 - request_id, prompt, params, arrival_time, lora_request,
232 - tokenization_kwargs, trace_headers, priority)
230 + request = self.processor.process_inputs(request_id, prompt, params,
231 + arrival_time, lora_request,
232 + tokenization_kwargs,
233 + trace_headers, priority)
234 + prompt_text = prompt if isinstance(prompt,
235 + str) else prompt.get("prompt")
236
237 n = params.n if isinstance(params, SamplingParams) else 1
238
239 if n == 1:
240 # Make a new RequestState and queue.
238 - self.output_processor.add_request(request, prompt_str, None, 0)
241 + self.output_processor.add_request(request, prompt_text, None, 0)
242 # Add the request to EngineCore.
243 self.engine_core.add_request(request)
244 return
@@ -249,7 +252,7 @@ class LLMEngine:
252 child_request.sampling_params = params
253
254 # Make a new RequestState and queue.
252 - self.output_processor.add_request(child_request, prompt_str,
255 + self.output_processor.add_request(child_request, prompt_text,
256 parent_req, idx)
257 # Add the request to EngineCore.
258 self.engine_core.add_request(child_request)
vllm/v1/engine/processor.py
+2 -6
@@ -334,9 +334,7 @@ class Processor:
334 trace_headers: Optional[Mapping[str, str]] = None,
335 priority: int = 0,
336 data_parallel_rank: Optional[int] = None,
337 - ) -> tuple[Optional[str], EngineCoreRequest]:
338 -
339 - # TODO(woosuk): Support pooling models.
337 + ) -> EngineCoreRequest:
338 self._validate_lora(lora_request)
339 self._validate_params(params)
340
@@ -395,8 +393,6 @@ class Processor:
393 # discriminated unions of TypedDicts, because of how it handles
394 # inheritance of TypedDict. If we explicitly extract the items we want
395 # we can avoid type errors from using `dict.get` later in the method.
398 - prompt_str: Optional[str] = None if decoder_inputs[
399 - "type"] == "embeds" else decoder_inputs.get("prompt")
396 prompt_token_ids = decoder_inputs[
397 "prompt_token_ids"] if decoder_inputs["type"] != "embeds" else None
398 prompt_embeds = decoder_inputs["prompt_embeds"] if decoder_inputs[
@@ -442,7 +438,7 @@ class Processor:
438 identifier=decoder_mm_hashes[modality][idx],
439 mm_position=decoder_mm_positions[modality][idx]))
440
445 - return prompt_str, EngineCoreRequest(
441 + return EngineCoreRequest(
442 request_id=request_id,
443 prompt_token_ids=prompt_token_ids,
444 prompt_embeds=prompt_embeds,