A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Renderer] Define `render_cmpl` and `render_chat` (#34039)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 7, 2026 at 21:24 UTC
edb359cce460de499331abb8f9c202aa40bbb335
7 files changed
+150
-75
tests/models/multimodal/generation/test_phi4mm.py
-2
@@ -170,8 +170,6 @@ def run_test(
170
@pytest.mark.parametrize(
171
"size_factors",
172
[
173
- # No image
174
- [],
173
# Single-scale
174
[1.0],
175
# Single-scale, batched
tests/models/multimodal/generation/test_qwen2_vl.py
-1
@@ -375,7 +375,6 @@ def test_qwen2_vl_image_embeddings_input(
375
@pytest.mark.parametrize(
376
"size_factors",
377
[
378
- [],
378
# Single-scale
379
[0.5],
380
# Single-scale, batched
tests/models/quantization/test_awq.py
-2
@@ -100,8 +100,6 @@ def run_awq_test(
100
@pytest.mark.parametrize(
101
"size_factors",
102
[
103
- # No image
104
- [],
103
# Single-scale
104
[1.0],
105
# Single-scale, batched
vllm/entrypoints/llm.py
+14
-28
@@ -73,7 +73,7 @@ from vllm.outputs import (
73
from vllm.platforms import current_platform
74
from vllm.pooling_params import PoolingParams
75
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
76
-from vllm.renderers.inputs import DictPrompt, SingletonDictPrompt, TokPrompt
76
+from vllm.renderers.inputs import DictPrompt, TokPrompt
77
from vllm.renderers.inputs.preprocess import (
78
conversation_to_seq,
79
extract_prompt_components,
@@ -805,7 +805,7 @@ class LLM:
805
self,
806
prompts: Sequence[PromptType],
807
tokenization_kwargs: dict[str, Any] | None = None,
808
- ) -> list[DictPrompt | TokPrompt]:
808
+ ) -> Sequence[DictPrompt | TokPrompt]:
809
"""
810
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
811
a format that can be passed to `_add_request`.
@@ -819,22 +819,12 @@ class LLM:
819
renderer = self.llm_engine.renderer
820
model_config = self.model_config
821
822
+ parsed_prompts = [
823
+ parse_model_prompt(model_config, prompt) for prompt in prompts
824
+ ]
825
tok_params = self._get_cmpl_tok_params(tokenization_kwargs)
826
824
- engine_prompts = list[DictPrompt | TokPrompt]()
825
- for prompt in prompts:
826
- parsed_prompt = parse_model_prompt(model_config, prompt)
827
- in_prompt = renderer.render_prompt(parsed_prompt)
828
-
829
- # Some MM models have non-default `add_special_tokens`
830
- # TODO: Move multi-modal processor into tokenization
831
- engine_prompts.append(
832
- in_prompt
833
- if model_config.is_multimodal_model
834
- else renderer.tokenize_prompt(in_prompt, tok_params)
835
- )
836
-
837
- return engine_prompts
827
+ return renderer.render_cmpl(parsed_prompts, tok_params)
828
829
def _get_chat_tok_params(self, tokenization_kwargs: dict[str, Any] | None):
830
model_config = self.model_config
@@ -857,7 +847,7 @@ class LLM:
847
tools: list[dict[str, Any]] | None = None,
848
tokenization_kwargs: dict[str, Any] | None = None,
849
mm_processor_kwargs: dict[str, Any] | None = None,
860
- ) -> list[DictPrompt | TokPrompt]:
850
+ ) -> Sequence[TokPrompt]:
851
"""
852
Convert a list of conversations into prompts so that they can then
853
be used as input for other LLM APIs.
@@ -885,16 +875,12 @@ class LLM:
875
)
876
tok_params = self._get_chat_tok_params(tokenization_kwargs)
877
888
- engine_prompts = list[DictPrompt | TokPrompt]()
889
- for conversation in conversations:
890
- _, in_prompt = renderer.render_messages(conversation, chat_params)
891
- if mm_processor_kwargs is not None:
892
- target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore
893
- "encoder_prompt", in_prompt
894
- )
895
- target_prompt["mm_processor_kwargs"] = mm_processor_kwargs # type: ignore
896
-
897
- engine_prompts.append(renderer.tokenize_prompt(in_prompt, tok_params))
878
+ _, engine_prompts = renderer.render_chat(
879
+ conversations,
880
+ chat_params,
881
+ tok_params,
882
+ prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
883
+ )
884
885
return engine_prompts
886
@@ -1743,7 +1729,7 @@ class LLM:
1729
# TODO: Remove this after deprecating `param.truncate_prompt_tokens`
1730
# Then, move the code from the `else` block to the top and let
1731
# `self._preprocess_completion` handle prompt normalization
1746
- engine_prompts = [
1732
+ engine_prompts: Sequence[DictPrompt | TokPrompt] = [
1733
engine_prompt
1734
for prompt, param in zip(seq_prompts, seq_params)
1735
for engine_prompt in self._preprocess_completion(
vllm/entrypoints/openai/engine/serving.py
+19
-32
@@ -106,7 +106,6 @@ from vllm.pooling_params import PoolingParams
106
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
107
from vllm.renderers.inputs import TokPrompt
108
from vllm.renderers.inputs.preprocess import (
109
- SingletonDictPrompt,
109
extract_prompt_components,
110
extract_prompt_len,
111
parse_model_prompt,
@@ -963,8 +962,6 @@ class OpenAIServing:
962
renderer = self.renderer
963
model_config = self.model_config
964
966
- tok_params = request.build_tok_params(model_config)
967
-
965
prompts = list[SingletonPrompt | bytes]()
966
if prompt_embeds is not None: # embeds take higher priority
967
prompts.extend(prompt_to_seq(prompt_embeds))
@@ -979,22 +976,17 @@ class OpenAIServing:
976
)
977
for prompt in prompts
978
]
982
- in_prompts = await renderer.render_prompts_async(parsed_prompts)
983
-
984
- extra_items = {
985
- k: v
986
- for k in ("mm_processor_kwargs", "cache_salt")
987
- if (v := getattr(request, k, None)) is not None
988
- }
989
- for in_prompt in in_prompts:
990
- target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore
991
- "encoder_prompt", in_prompt
992
- )
993
- target_prompt.update(extra_items) # type: ignore
994
-
995
- engine_prompts = await renderer.tokenize_prompts_async(in_prompts, tok_params)
979
+ tok_params = request.build_tok_params(model_config)
980
997
- return engine_prompts
981
+ return await renderer.render_cmpl_async(
982
+ parsed_prompts,
983
+ tok_params,
984
+ prompt_extras={
985
+ k: v
986
+ for k in ("mm_processor_kwargs", "cache_salt")
987
+ if (v := getattr(request, k, None)) is not None
988
+ },
989
+ )
990
991
async def _preprocess_chat(
992
self,
@@ -1023,22 +1015,17 @@ class OpenAIServing:
1015
default_template, default_template_content_format
1016
).with_defaults(default_template_kwargs)
1017
1026
- conversation, in_prompt = await renderer.render_messages_async(
1027
- messages, chat_params
1028
- )
1029
- target_prompt: SingletonDictPrompt = in_prompt.get( # type: ignore
1030
- "encoder_prompt", in_prompt
1018
+ (conversation,), (engine_prompt,) = await renderer.render_chat_async(
1019
+ [messages],
1020
+ chat_params,
1021
+ tok_params,
1022
+ prompt_extras={
1023
+ k: v
1024
+ for k in ("mm_processor_kwargs", "cache_salt")
1025
+ if (v := getattr(request, k, None)) is not None
1026
+ },
1027
)
1028
1033
- extra_items = {
1034
- k: v
1035
- for k in ("mm_processor_kwargs", "cache_salt")
1036
- if (v := getattr(request, k, None)) is not None
1037
- }
1038
- target_prompt.update(extra_items) # type: ignore
1039
-
1040
- engine_prompt = await renderer.tokenize_prompt_async(target_prompt, tok_params)
1041
-
1029
# tool parsing is done only if a tool_parser has been set and if
1030
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
1031
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
vllm/renderers/inputs/preprocess.py
+10
-10
@@ -225,16 +225,20 @@ class PromptComponents(NamedTuple):
225
embeds: "torch.Tensor | None" = None
226
227
228
-def extract_prompt_components(
229
- model_config: "ModelConfig",
230
- prompt: object,
231
-) -> PromptComponents:
232
- target_prompt = (
228
+def extract_target_prompt(model_config: "ModelConfig", prompt: object):
229
+ return (
230
parse_enc_dec_prompt(prompt)["encoder_prompt"]
231
if model_config.is_encoder_decoder
232
else parse_dec_only_prompt(prompt)
233
)
234
235
+
236
+def extract_prompt_components(
237
+ model_config: "ModelConfig",
238
+ prompt: object,
239
+) -> PromptComponents:
240
+ target_prompt = extract_target_prompt(model_config, prompt)
241
+
242
return PromptComponents(
243
text=target_prompt.get("prompt"),
244
token_ids=target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]
@@ -243,11 +247,7 @@ def extract_prompt_components(
247
248
249
def extract_prompt_len(model_config: "ModelConfig", prompt: object):
246
- target_prompt = (
247
- parse_enc_dec_prompt(prompt)["encoder_prompt"]
248
- if model_config.is_encoder_decoder
249
- else parse_dec_only_prompt(prompt)
250
- )
250
+ target_prompt = extract_target_prompt(model_config, prompt)
251
252
return length_from_prompt_token_ids_or_embeds(
253
target_prompt.get("prompt_token_ids"), # type: ignore[arg-type]
vllm/renderers/protocol.py
+107
@@ -16,6 +16,7 @@ from .inputs import (
16
EncoderDecoderTokPrompt,
17
TokPrompt,
18
)
19
+from .inputs.preprocess import extract_target_prompt
20
from .params import ChatParams, TokenizeParams
21
22
if TYPE_CHECKING:
@@ -277,3 +278,109 @@ class BaseRenderer(ABC):
278
return await asyncio.gather(
279
*(self.tokenize_prompt_async(prompt, params) for prompt in prompts)
280
)
281
+
282
+ # Step 3: Add extra keys to the prompts
283
+ def _apply_prompt_extras(
284
+ self,
285
+ prompts: Sequence[DictPrompt | TokPrompt],
286
+ prompt_extras: dict[str, Any] | None,
287
+ ):
288
+ if not prompt_extras:
289
+ return
290
+
291
+ for prompt in prompts:
292
+ target_prompt = extract_target_prompt(self.config, prompt)
293
+ target_prompt.update(prompt_extras) # type: ignore[arg-type]
294
+
295
+ # Top-level methods
296
+ def render_cmpl(
297
+ self,
298
+ prompts: Sequence[DictPrompt | bytes],
299
+ tok_params: TokenizeParams,
300
+ *,
301
+ prompt_extras: dict[str, Any] | None = None,
302
+ ):
303
+ dict_prompts = self.render_prompts(prompts)
304
+
305
+ # NOTE: Some MM models have non-default `add_special_tokens`
306
+ # so we handle tokenization in multi-modal processor
307
+ if self.config.is_multimodal_model:
308
+ self._apply_prompt_extras(dict_prompts, prompt_extras)
309
+ return dict_prompts
310
+
311
+ tok_prompts = self.tokenize_prompts(dict_prompts, tok_params)
312
+
313
+ self._apply_prompt_extras(tok_prompts, prompt_extras)
314
+
315
+ # TODO: Apply multi-modal processor
316
+ return tok_prompts
317
+
318
+ async def render_cmpl_async(
319
+ self,
320
+ prompts: Sequence[DictPrompt | bytes],
321
+ tok_params: TokenizeParams,
322
+ *,
323
+ prompt_extras: dict[str, Any] | None = None,
324
+ ):
325
+ dict_prompts = await self.render_prompts_async(prompts)
326
+
327
+ # NOTE: MM data cannot be passed to online Completions API
328
+ # so we don't have the special case that is in the offline version
329
+ tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params)
330
+
331
+ self._apply_prompt_extras(tok_prompts, prompt_extras)
332
+
333
+ # TODO: Apply multi-modal processor
334
+ return tok_prompts
335
+
336
+ def render_chat(
337
+ self,
338
+ conversations: Sequence[list["ChatCompletionMessageParam"]],
339
+ chat_params: ChatParams,
340
+ tok_params: TokenizeParams,
341
+ *,
342
+ prompt_extras: dict[str, Any] | None = None,
343
+ ):
344
+ rendered = [
345
+ self.render_messages(conversation, chat_params)
346
+ for conversation in conversations
347
+ ]
348
+
349
+ out_conversations = list[list["ConversationMessage"]]()
350
+ dict_prompts = list[DictPrompt]()
351
+ for conv, prompt in rendered:
352
+ out_conversations.append(conv)
353
+ dict_prompts.append(prompt)
354
+
355
+ tok_prompts = self.tokenize_prompts(dict_prompts, tok_params)
356
+
357
+ self._apply_prompt_extras(tok_prompts, prompt_extras)
358
+
359
+ # TODO: Apply multi-modal processor
360
+ return out_conversations, tok_prompts
361
+
362
+ async def render_chat_async(
363
+ self,
364
+ conversations: Sequence[list["ChatCompletionMessageParam"]],
365
+ chat_params: ChatParams,
366
+ tok_params: TokenizeParams,
367
+ *,
368
+ prompt_extras: dict[str, Any] | None = None,
369
+ ):
370
+ rendered = [
371
+ self.render_messages_async(conversation, chat_params)
372
+ for conversation in conversations
373
+ ]
374
+
375
+ out_conversations = list[list["ConversationMessage"]]()
376
+ dict_prompts = list[DictPrompt]()
377
+ for conv, prompt in await asyncio.gather(*rendered):
378
+ out_conversations.append(conv)
379
+ dict_prompts.append(prompt)
380
+
381
+ tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params)
382
+
383
+ self._apply_prompt_extras(tok_prompts, prompt_extras)
384
+
385
+ # TODO: Apply multi-modal processor
386
+ return out_conversations, tok_prompts