A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 07:30 UTC
Explicitly set `return_dict` for `apply_chat_template` (#33372)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jan 30, 2026 at 07:27 UTC
9432ed8c7e158def084e0770fd02838292bf57e4
11 files changed
+22
-10
examples/offline_inference/prompt_embed_inference.py
+2
-2
@@ -38,8 +38,8 @@ def get_prompt_embeds(
38
embedding_layer: torch.nn.Module,
39
):
40
token_ids = tokenizer.apply_chat_template(
41
- chat, add_generation_prompt=True, return_tensors="pt"
42
- )
41
+ chat, add_generation_prompt=True, return_tensors="pt", return_dict=True
42
+ ).input_ids
43
prompt_embeds = embedding_layer(token_ids).squeeze(0)
44
return prompt_embeds
45
examples/online_serving/prompt_embed_inference_with_openai_client.py
+2
-2
@@ -49,8 +49,8 @@ def main():
49
# Refer to the HuggingFace repo for the correct format to use
50
chat = [{"role": "user", "content": "Please tell me about the capital of France."}]
51
token_ids = tokenizer.apply_chat_template(
52
- chat, add_generation_prompt=True, return_tensors="pt"
53
- )
52
+ chat, add_generation_prompt=True, return_tensors="pt", return_dict=True
53
+ ).input_ids
54
55
embedding_layer = transformers_model.get_input_embeddings()
56
prompt_embeds = embedding_layer(token_ids).squeeze(0)
examples/online_serving/token_generation_client.py
+2
-1
@@ -27,7 +27,8 @@ def main(client):
27
messages,
28
add_generation_prompt=True,
29
enable_thinking=False,
30
- )
30
+ return_dict=True,
31
+ ).input_ids
32
payload = {
33
"model": MODEL_NAME,
34
"token_ids": token_ids,
tests/entrypoints/openai/test_serving_tokens.py
+6
-3
@@ -92,7 +92,8 @@ async def test_same_response_as_chat_completions(client, tokenizer, messages):
92
messages,
93
add_generation_prompt=True,
94
enable_thinking=False, # default with Qwen3
95
- )
95
+ return_dict=True, # default with Transformers v5
96
+ ).input_ids
97
98
for ignore_eos in [True, False]:
99
payload = {
@@ -155,7 +156,8 @@ async def test_stop_string_workflow(client, tokenizer, messages):
156
messages,
157
add_generation_prompt=True,
158
enable_thinking=False, # default with Qwen3
158
- )
159
+ return_dict=True, # default with Transformers v5
160
+ ).input_ids
161
payload = {
162
"model": MODEL_NAME,
163
"token_ids": token_ids,
@@ -251,7 +253,8 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
253
messages,
254
add_generation_prompt=True,
255
enable_thinking=False, # default with Qwen3
254
- )
256
+ return_dict=True, # default with Transformers v5
257
+ ).input_ids
258
payload = {
259
"model": "Alice",
260
"token_ids": token_ids,
vllm/model_executor/models/isaac.py
+1
@@ -759,6 +759,7 @@ class IsaacProcessor:
759
# Regular text message
760
processed_messages.append(message)
761
762
+ kwargs["return_dict"] = False
763
return self.tokenizer.apply_chat_template(
764
processed_messages,
765
tokenize=tokenize,
vllm/renderers/deepseek_v32.py
+2
@@ -70,6 +70,7 @@ class DeepseekV32Renderer(RendererLike):
70
content_format="string",
71
)
72
73
+ kwargs["return_dict"] = False
74
prompt_raw = tokenizer.apply_chat_template(
75
conversation=conversation,
76
messages=messages,
@@ -100,6 +101,7 @@ class DeepseekV32Renderer(RendererLike):
101
content_format="string",
102
)
103
104
+ kwargs["return_dict"] = False
105
prompt_raw = tokenizer.apply_chat_template(
106
conversation=conversation,
107
messages=messages,
vllm/renderers/grok2.py
+2
@@ -70,6 +70,7 @@ class Grok2Renderer(RendererLike):
70
content_format="string",
71
)
72
73
+ kwargs["return_dict"] = False
74
prompt_raw = tokenizer.apply_chat_template(
75
conversation=conversation,
76
messages=messages,
@@ -100,6 +101,7 @@ class Grok2Renderer(RendererLike):
101
content_format="string",
102
)
103
104
+ kwargs["return_dict"] = False
105
prompt_raw = tokenizer.apply_chat_template(
106
conversation=conversation,
107
messages=messages,
vllm/renderers/hf.py
+1
@@ -465,6 +465,7 @@ def safe_apply_chat_template(
465
chat_template=chat_template,
466
chat_template_kwargs=kwargs,
467
)
468
+ resolved_kwargs["return_dict"] = False
469
470
try:
471
return tokenizer.apply_chat_template(
vllm/tokenizers/grok2.py
+1
@@ -432,6 +432,7 @@ class Grok2Tokenizer(TokenizerLike):
432
raise ValueError(
433
"No chat template available. Provide `chat_template` explicitly."
434
)
435
+ kwargs["return_dict"] = False
436
prompt = hf_chat_utils.apply_chat_template(
437
conversation=messages,
438
chat_template=template,
vllm/transformers_utils/processors/hunyuan_vl.py
+2
-2
@@ -148,8 +148,8 @@ class HunYuanVLProcessor(ProcessorMixin):
148
assert 0
149
150
def apply_chat_template(self, *args, **kwargs):
151
- token_ids = self.tokenizer.apply_chat_template(*args, **kwargs)
152
- return token_ids
151
+ kwargs["return_dict"] = False
152
+ return self.tokenizer.apply_chat_template(*args, **kwargs)
153
154
def get_imgs_pos(self, doc_ids):
155
doc_ids = np.array(doc_ids, dtype=np.int64)
vllm/transformers_utils/processors/qwen3_asr.py
+1
@@ -213,6 +213,7 @@ class Qwen3ASRProcessor(ProcessorMixin):
213
return list(_iter())
214
215
def apply_chat_template(self, conversations, chat_template=None, **kwargs):
216
+ kwargs["return_dict"] = False
217
return super().apply_chat_template(conversations, chat_template, **kwargs)
218
219
@property