A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Frontend] Automatic detection of chat content format from AST (#9919)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 16, 2024 at 13:35 UTC
32e46e000f77499f4dd7c0bed194e33856f2df24
16 files changed
+788
-350
docs/source/serving/openai_compatible_server.md
+13
-5
@@ -172,12 +172,20 @@ completion = client.chat.completions.create(
172
]
173
)
174
```
175
-Most chat templates for LLMs expect the `content` to be a `string` but there are some newer models like
176
-`meta-llama/Llama-Guard-3-1B` that expect the content to be parsed with the new OpenAI spec. In order to choose which
177
-format the content needs to be parsed in by vLLM, please use the `--chat-template-text-format` argument to specify
178
-between `string` or `openai`. The default value is `string` and vLLM internally converts both spec formats to match
179
-this, unless explicitly specified.
175
176
+Most chat templates for LLMs expect the `content` field to be a string, but there are some newer models like
177
+`meta-llama/Llama-Guard-3-1B` that expect the content to be formatted according to the OpenAI schema in the
178
+request. vLLM provides best-effort support to detect this automatically, which is logged as a string like
179
+*"Detected the chat template content format to be..."*, and internally converts incoming requests to match
180
+the detected format, which can be one of:
181
+
182
+- `"string"`: A string.
183
+ - Example: `"Hello world"`
184
+- `"openai"`: A list of dictionaries, similar to OpenAI schema.
185
+ - Example: `[{"type": "text", "text": "Hello world!"}]`
186
+
187
+If the result is not what you expect, you can set the `--chat-template-content-format` CLI argument
188
+to override which format to use.
189
190
## Command line arguments for the server
191
tests/entrypoints/openai/test_serving_chat.py
+2
-1
@@ -26,7 +26,6 @@ class MockModelConfig:
26
tokenizer = MODEL_NAME
27
trust_remote_code = False
28
tokenizer_mode = "auto"
29
- chat_template_text_format = "string"
29
max_model_len = 100
30
tokenizer_revision = None
31
multimodal_config = MultiModalConfig()
@@ -49,6 +48,7 @@ async def _async_serving_chat_init():
48
BASE_MODEL_PATHS,
49
response_role="assistant",
50
chat_template=CHAT_TEMPLATE,
51
+ chat_template_content_format="auto",
52
lora_modules=None,
53
prompt_adapters=None,
54
request_logger=None)
@@ -70,6 +70,7 @@ def test_serving_chat_should_set_correct_max_tokens():
70
BASE_MODEL_PATHS,
71
response_role="assistant",
72
chat_template=CHAT_TEMPLATE,
73
+ chat_template_content_format="auto",
74
lora_modules=None,
75
prompt_adapters=None,
76
request_logger=None)
tests/entrypoints/test_chat_utils.py
+390
-229
@@ -6,15 +6,24 @@ from PIL import Image
6
7
from vllm.assets.image import ImageAsset
8
from vllm.config import ModelConfig
9
-from vllm.entrypoints.chat_utils import (parse_chat_messages,
10
- parse_chat_messages_futures)
9
+from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
10
+ parse_chat_messages,
11
+ parse_chat_messages_futures,
12
+ resolve_chat_template_content_format)
13
from vllm.entrypoints.llm import apply_hf_chat_template
14
from vllm.multimodal import MultiModalDataDict
15
from vllm.multimodal.utils import encode_image_base64
16
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
17
18
+from ..utils import VLLM_PATH
19
+
20
+EXAMPLES_DIR = VLLM_PATH / "examples"
21
+
22
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
23
+ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_3"
24
+QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
25
MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
26
+LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
27
28
29
@pytest.fixture(scope="function")
@@ -26,7 +35,6 @@ def phi3v_model_config():
35
trust_remote_code=True,
36
dtype="bfloat16",
37
seed=0,
29
- chat_template_text_format="string",
38
limit_mm_per_prompt={
39
"image": 2,
40
})
@@ -94,19 +102,24 @@ def test_parse_chat_messages_single_image(
102
phi3v_tokenizer,
103
image_url,
104
):
97
- conversation, mm_data = parse_chat_messages([{
98
- "role":
99
- "user",
100
- "content": [{
101
- "type": "image_url",
102
- "image_url": {
103
- "url": image_url
104
- }
105
- }, {
106
- "type": "text",
107
- "text": "What's in the image?"
108
- }]
109
- }], phi3v_model_config, phi3v_tokenizer)
105
+ conversation, mm_data = parse_chat_messages(
106
+ [{
107
+ "role":
108
+ "user",
109
+ "content": [{
110
+ "type": "image_url",
111
+ "image_url": {
112
+ "url": image_url
113
+ }
114
+ }, {
115
+ "type": "text",
116
+ "text": "What's in the image?"
117
+ }]
118
+ }],
119
+ phi3v_model_config,
120
+ phi3v_tokenizer,
121
+ content_format="string",
122
+ )
123
124
assert conversation == [{
125
"role": "user",
@@ -121,19 +134,24 @@ async def test_parse_chat_messages_single_image_async(
134
phi3v_tokenizer,
135
image_url,
136
):
124
- conversation, mm_future = parse_chat_messages_futures([{
125
- "role":
126
- "user",
127
- "content": [{
128
- "type": "image_url",
129
- "image_url": {
130
- "url": image_url
131
- }
132
- }, {
133
- "type": "text",
134
- "text": "What's in the image?"
135
- }]
136
- }], phi3v_model_config, phi3v_tokenizer)
137
+ conversation, mm_future = parse_chat_messages_futures(
138
+ [{
139
+ "role":
140
+ "user",
141
+ "content": [{
142
+ "type": "image_url",
143
+ "image_url": {
144
+ "url": image_url
145
+ }
146
+ }, {
147
+ "type": "text",
148
+ "text": "What's in the image?"
149
+ }]
150
+ }],
151
+ phi3v_model_config,
152
+ phi3v_tokenizer,
153
+ content_format="string",
154
+ )
155
156
assert conversation == [{
157
"role": "user",
@@ -147,24 +165,29 @@ def test_parse_chat_messages_multiple_images(
165
phi3v_tokenizer,
166
image_url,
167
):
150
- conversation, mm_data = parse_chat_messages([{
151
- "role":
152
- "user",
153
- "content": [{
154
- "type": "image_url",
155
- "image_url": {
156
- "url": image_url
157
- }
158
- }, {
159
- "type": "image_url",
160
- "image_url": {
161
- "url": image_url
162
- }
163
- }, {
164
- "type": "text",
165
- "text": "What's in these images?"
166
- }]
167
- }], phi3v_model_config, phi3v_tokenizer)
168
+ conversation, mm_data = parse_chat_messages(
169
+ [{
170
+ "role":
171
+ "user",
172
+ "content": [{
173
+ "type": "image_url",
174
+ "image_url": {
175
+ "url": image_url
176
+ }
177
+ }, {
178
+ "type": "image_url",
179
+ "image_url": {
180
+ "url": image_url
181
+ }
182
+ }, {
183
+ "type": "text",
184
+ "text": "What's in these images?"
185
+ }]
186
+ }],
187
+ phi3v_model_config,
188
+ phi3v_tokenizer,
189
+ content_format="string",
190
+ )
191
192
assert conversation == [{
193
"role":
@@ -181,24 +204,29 @@ async def test_parse_chat_messages_multiple_images_async(
204
phi3v_tokenizer,
205
image_url,
206
):
184
- conversation, mm_future = parse_chat_messages_futures([{
185
- "role":
186
- "user",
187
- "content": [{
188
- "type": "image_url",
189
- "image_url": {
190
- "url": image_url
191
- }
192
- }, {
193
- "type": "image_url",
194
- "image_url": {
195
- "url": image_url
196
- }
197
- }, {
198
- "type": "text",
199
- "text": "What's in these images?"
200
- }]
201
- }], phi3v_model_config, phi3v_tokenizer)
207
+ conversation, mm_future = parse_chat_messages_futures(
208
+ [{
209
+ "role":
210
+ "user",
211
+ "content": [{
212
+ "type": "image_url",
213
+ "image_url": {
214
+ "url": image_url
215
+ }
216
+ }, {
217
+ "type": "image_url",
218
+ "image_url": {
219
+ "url": image_url
220
+ }
221
+ }, {
222
+ "type": "text",
223
+ "text": "What's in these images?"
224
+ }]
225
+ }],
226
+ phi3v_model_config,
227
+ phi3v_tokenizer,
228
+ content_format="string",
229
+ )
230
231
assert conversation == [{
232
"role":
@@ -214,27 +242,31 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
242
phi3v_tokenizer,
243
image_url,
244
):
217
- conversation, mm_data = parse_chat_messages([{
218
- "role":
219
- "user",
220
- "content": [{
221
- "type": "image_url",
222
- "image_url": {
223
- "url": image_url
224
- }
225
- }, {
226
- "type": "image_url",
227
- "image_url": {
228
- "url": image_url
229
- }
230
- }, {
231
- "type":
232
- "text",
233
- "text":
234
- "What's in <|image_1|> and how does it compare to <|image_2|>?"
235
- }]
236
- }], phi3v_model_config, phi3v_tokenizer)
237
-
245
+ conversation, mm_data = parse_chat_messages(
246
+ [{
247
+ "role":
248
+ "user",
249
+ "content": [{
250
+ "type": "image_url",
251
+ "image_url": {
252
+ "url": image_url
253
+ }
254
+ }, {
255
+ "type": "image_url",
256
+ "image_url": {
257
+ "url": image_url
258
+ }
259
+ }, {
260
+ "type":
261
+ "text",
262
+ "text":
263
+ "What's in <|image_1|> and how does it compare to <|image_2|>?"
264
+ }]
265
+ }],
266
+ phi3v_model_config,
267
+ phi3v_tokenizer,
268
+ content_format="string",
269
+ )
270
assert conversation == [{
271
"role":
272
"user",
@@ -249,26 +281,35 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
281
phi3v_tokenizer,
282
image_url,
283
):
252
- conversation, mm_data = parse_chat_messages([{
253
- "role":
254
- "user",
255
- "content": [{
256
- "type": "image_url",
257
- "image_url": {
258
- "url": image_url
259
- }
260
- }, {
261
- "type": "image_url",
262
- "image_url": {
263
- "url": image_url
264
- }
265
- }, {
266
- "type":
267
- "text",
268
- "text":
269
- "What's in <|image_1|> and how does it compare to the other one?"
270
- }]
271
- }], phi3v_model_config, phi3v_tokenizer)
284
+ conversation, mm_data = parse_chat_messages(
285
+ [{
286
+ "role":
287
+ "user",
288
+ "content": [
289
+ {
290
+ "type": "image_url",
291
+ "image_url": {
292
+ "url": image_url
293
+ }
294
+ },
295
+ {
296
+ "type": "image_url",
297
+ "image_url": {
298
+ "url": image_url
299
+ }
300
+ },
301
+ {
302
+ "type":
303
+ "text",
304
+ "text":
305
+ "What's in <|image_1|> and how does it compare to the other one?" # noqa: E501
306
+ }
307
+ ]
308
+ }],
309
+ phi3v_model_config,
310
+ phi3v_tokenizer,
311
+ content_format="string",
312
+ )
313
314
assert conversation == [{
315
"role":
@@ -285,34 +326,39 @@ def test_parse_chat_messages_multiple_images_across_messages(
326
phi3v_tokenizer,
327
image_url,
328
):
288
- conversation, mm_data = parse_chat_messages([{
289
- "role":
290
- "user",
291
- "content": [{
292
- "type": "image_url",
293
- "image_url": {
294
- "url": image_url
295
- }
329
+ conversation, mm_data = parse_chat_messages(
330
+ [{
331
+ "role":
332
+ "user",
333
+ "content": [{
334
+ "type": "image_url",
335
+ "image_url": {
336
+ "url": image_url
337
+ }
338
+ }, {
339
+ "type": "text",
340
+ "text": "What's in this image?"
341
+ }]
342
}, {
297
- "type": "text",
298
- "text": "What's in this image?"
299
- }]
300
- }, {
301
- "role": "assistant",
302
- "content": "Some stuff."
303
- }, {
304
- "role":
305
- "user",
306
- "content": [{
307
- "type": "image_url",
308
- "image_url": {
309
- "url": image_url
310
- }
343
+ "role": "assistant",
344
+ "content": "Some stuff."
345
}, {
312
- "type": "text",
313
- "text": "What about this one?"
314
- }]
315
- }], phi3v_model_config, phi3v_tokenizer)
346
+ "role":
347
+ "user",
348
+ "content": [{
349
+ "type": "image_url",
350
+ "image_url": {
351
+ "url": image_url
352
+ }
353
+ }, {
354
+ "type": "text",
355
+ "text": "What about this one?"
356
+ }]
357
+ }],
358
+ phi3v_model_config,
359
+ phi3v_tokenizer,
360
+ content_format="string",
361
+ )
362
363
assert conversation == [
364
{
@@ -335,7 +381,6 @@ def test_parse_chat_messages_context_text_format(
381
phi3v_model_config,
382
phi3v_tokenizer,
383
):
338
- phi3v_model_config.chat_template_text_format = "openai"
384
conversation, mm_data = parse_chat_messages(
385
[{
386
"role": "user",
@@ -349,7 +394,11 @@ def test_parse_chat_messages_context_text_format(
394
}, {
395
"role": "user",
396
"content": "What about this one?"
352
- }], phi3v_model_config, phi3v_tokenizer)
397
+ }],
398
+ phi3v_model_config,
399
+ phi3v_tokenizer,
400
+ content_format="openai",
401
+ )
402
403
assert conversation == [
404
{
@@ -389,29 +438,34 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
438
ValueError,
439
match="At most 2 image\\(s\\) may be provided in one request\\."
440
):
392
- parse_chat_messages([{
393
- "role":
394
- "user",
395
- "content": [{
396
- "type": "image_url",
397
- "image_url": {
398
- "url": image_url
399
- }
400
- }, {
401
- "type": "image_url",
402
- "image_url": {
403
- "url": image_url
404
- }
405
- }, {
406
- "type": "image_url",
407
- "image_url": {
408
- "url": image_url
409
- }
410
- }, {
411
- "type": "text",
412
- "text": "What's in these images?"
413
- }]
414
- }], phi3v_model_config, phi3v_tokenizer)
441
+ parse_chat_messages(
442
+ [{
443
+ "role":
444
+ "user",
445
+ "content": [{
446
+ "type": "image_url",
447
+ "image_url": {
448
+ "url": image_url
449
+ }
450
+ }, {
451
+ "type": "image_url",
452
+ "image_url": {
453
+ "url": image_url
454
+ }
455
+ }, {
456
+ "type": "image_url",
457
+ "image_url": {
458
+ "url": image_url
459
+ }
460
+ }, {
461
+ "type": "text",
462
+ "text": "What's in these images?"
463
+ }]
464
+ }],
465
+ phi3v_model_config,
466
+ phi3v_tokenizer,
467
+ content_format="string",
468
+ )
469
470
471
def test_parse_chat_messages_rejects_too_many_images_across_messages(
@@ -427,39 +481,44 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
481
ValueError,
482
match="At most 2 image\\(s\\) may be provided in one request\\."
483
):
430
- parse_chat_messages([{
431
- "role":
432
- "user",
433
- "content": [{
434
- "type": "image_url",
435
- "image_url": {
436
- "url": image_url
437
- }
484
+ parse_chat_messages(
485
+ [{
486
+ "role":
487
+ "user",
488
+ "content": [{
489
+ "type": "image_url",
490
+ "image_url": {
491
+ "url": image_url
492
+ }
493
+ }, {
494
+ "type": "text",
495
+ "text": "What's in this image?"
496
+ }]
497
}, {
439
- "type": "text",
440
- "text": "What's in this image?"
441
- }]
442
- }, {
443
- "role": "assistant",
444
- "content": "Some stuff."
445
- }, {
446
- "role":
447
- "user",
448
- "content": [{
449
- "type": "image_url",
450
- "image_url": {
451
- "url": image_url
452
- }
498
+ "role": "assistant",
499
+ "content": "Some stuff."
500
}, {
454
- "type": "image_url",
455
- "image_url": {
456
- "url": image_url
457
- }
458
- }, {
459
- "type": "text",
460
- "text": "What about these two?"
461
- }]
462
- }], phi3v_model_config, phi3v_tokenizer)
501
+ "role":
502
+ "user",
503
+ "content": [{
504
+ "type": "image_url",
505
+ "image_url": {
506
+ "url": image_url
507
+ }
508
+ }, {
509
+ "type": "image_url",
510
+ "image_url": {
511
+ "url": image_url
512
+ }
513
+ }, {
514
+ "type": "text",
515
+ "text": "What about these two?"
516
+ }]
517
+ }],
518
+ phi3v_model_config,
519
+ phi3v_tokenizer,
520
+ content_format="string",
521
+ )
522
523
524
def test_parse_chat_messages_multiple_images_uncommon_input(
@@ -467,17 +526,22 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
526
phi3v_tokenizer,
527
image_url,
528
):
470
- conversation, mm_data = parse_chat_messages([{
471
- "role":
472
- "user",
473
- "content": [
474
- "What's in these images?", {
475
- "image_url": image_url
476
- }, {
477
- "image_url": image_url
478
- }
479
- ]
480
- }], phi3v_model_config, phi3v_tokenizer)
529
+ conversation, mm_data = parse_chat_messages(
530
+ [{
531
+ "role":
532
+ "user",
533
+ "content": [
534
+ "What's in these images?", {
535
+ "image_url": image_url
536
+ }, {
537
+ "image_url": image_url
538
+ }
539
+ ]
540
+ }],
541
+ phi3v_model_config,
542
+ phi3v_tokenizer,
543
+ content_format="string",
544
+ )
545
546
assert conversation == [{
547
"role":
@@ -495,16 +559,21 @@ def test_mllama_single_image(
559
image_url,
560
):
561
"""Ensures that a single image is parsed correctly mllama."""
498
- conversation, mm_data = parse_chat_messages([{
499
- "role":
500
- "user",
501
- "content": [{
502
- 'type': 'text',
503
- 'text': 'The content of this image is:'
504
- }, {
505
- "image_url": image_url
506
- }]
507
- }], mllama_model_config, mllama_tokenizer)
562
+ conversation, mm_data = parse_chat_messages(
563
+ [{
564
+ "role":
565
+ "user",
566
+ "content": [{
567
+ 'type': 'text',
568
+ 'text': 'The content of this image is:'
569
+ }, {
570
+ "image_url": image_url
571
+ }]
572
+ }],
573
+ mllama_model_config,
574
+ mllama_tokenizer,
575
+ content_format="openai",
576
+ )
577
_assert_mm_data_is_image_input(mm_data, 1)
578
assert conversation == [{
579
'role':
@@ -524,26 +593,31 @@ def test_mllama_interleaved_images(
593
image_url,
594
):
595
"""Ensures that multiple image are parsed as interleaved dicts."""
527
- conversation, mm_data = parse_chat_messages([{
528
- "role":
529
- "user",
530
- "content": [
531
- {
532
- 'type': 'text',
533
- 'text': 'The content of the first image is:'
534
- },
535
- {
536
- "image_url": image_url
537
- },
538
- {
539
- 'type': 'text',
540
- 'text': 'The content of the second image is:'
541
- },
542
- {
543
- "image_url": image_url
544
- },
545
- ]
546
- }], mllama_model_config, mllama_tokenizer)
596
+ conversation, mm_data = parse_chat_messages(
597
+ [{
598
+ "role":
599
+ "user",
600
+ "content": [
601
+ {
602
+ 'type': 'text',
603
+ 'text': 'The content of the first image is:'
604
+ },
605
+ {
606
+ "image_url": image_url
607
+ },
608
+ {
609
+ 'type': 'text',
610
+ 'text': 'The content of the second image is:'
611
+ },
612
+ {
613
+ "image_url": image_url
614
+ },
615
+ ]
616
+ }],
617
+ mllama_model_config,
618
+ mllama_tokenizer,
619
+ content_format="openai",
620
+ )
621
_assert_mm_data_is_image_input(mm_data, 2)
622
assert conversation == [{
623
'role':
@@ -626,6 +700,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
700
vllm_conversation,
701
model_config,
702
tokenizer_group,
703
+ content_format="openai",
704
)
705
706
vllm_result = apply_hf_chat_template(
@@ -636,3 +711,89 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
711
)
712
713
assert hf_result == vllm_result
714
+
715
+
716
+# yapf: disable
717
+@pytest.mark.parametrize(
718
+ ("model", "expected_format"),
719
+ [(PHI3V_MODEL_ID, "string"),
720
+ (QWEN2VL_MODEL_ID, "openai"),
721
+ (ULTRAVOX_MODEL_ID, "string"),
722
+ (MLLAMA_MODEL_ID, "openai"),
723
+ (LLAMA_GUARD_MODEL_ID, "openai")],
724
+)
725
+# yapf: enable
726
+def test_resolve_content_format_hf_defined(model, expected_format):
727
+ tokenizer_group = TokenizerGroup(
728
+ model,
729
+ enable_lora=False,
730
+ max_num_seqs=5,
731
+ max_input_length=None,
732
+ )
733
+ tokenizer = tokenizer_group.tokenizer
734
+
735
+ chat_template = tokenizer.chat_template
736
+ assert isinstance(chat_template, str)
737
+
738
+ print("[TEXT]")
739
+ print(chat_template)
740
+ print("[AST]")
741
+ print(_try_extract_ast(chat_template))
742
+
743
+ resolved_format = resolve_chat_template_content_format(
744
+ None, # Test detecting the tokenizer's chat_template
745
+ "auto",
746
+ tokenizer,
747
+ )
748
+
749
+ assert resolved_format == expected_format
750
+
751
+
752
+# yapf: disable
753
+@pytest.mark.parametrize(
754
+ ("template_path", "expected_format"),
755
+ [("template_alpaca.jinja", "string"),
756
+ ("template_baichuan.jinja", "string"),
757
+ ("template_blip2.jinja", "string"),
758
+ ("template_chatglm.jinja", "string"),
759
+ ("template_chatglm2.jinja", "string"),
760
+ ("template_chatml.jinja", "string"),
761
+ ("template_falcon_180b.jinja", "string"),
762
+ ("template_falcon.jinja", "string"),
763
+ ("template_inkbot.jinja", "string"),
764
+ ("template_llava.jinja", "string"),
765
+ ("template_vlm2vec.jinja", "openai"),
766
+ ("tool_chat_template_granite_20b_fc.jinja", "string"),
767
+ ("tool_chat_template_hermes.jinja", "string"),
768
+ ("tool_chat_template_internlm2_tool.jinja", "string"),
769
+ ("tool_chat_template_llama3.1_json.jinja", "string"),
770
+ ("tool_chat_template_llama3.2_json.jinja", "string"),
771
+ ("tool_chat_template_mistral_parallel.jinja", "string"),
772
+ ("tool_chat_template_mistral.jinja", "string")],
773
+)
774
+# yapf: enable
775
+def test_resolve_content_format_examples(template_path, expected_format):
776
+ tokenizer_group = TokenizerGroup(
777
+ PHI3V_MODEL_ID,
778
+ enable_lora=False,
779
+ max_num_seqs=5,
780
+ max_input_length=None,
781
+ )
782
+ dummy_tokenizer = tokenizer_group.tokenizer
783
+ dummy_tokenizer.chat_template = None
784
+
785
+ chat_template = load_chat_template(EXAMPLES_DIR / template_path)
786
+ assert isinstance(chat_template, str)
787
+
788
+ print("[TEXT]")
789
+ print(chat_template)
790
+ print("[AST]")
791
+ print(_try_extract_ast(chat_template))
792
+
793
+ resolved_format = resolve_chat_template_content_format(
794
+ chat_template,
795
+ "auto",
796
+ dummy_tokenizer,
797
+ )
798
+
799
+ assert resolved_format == expected_format
vllm/config.py
-2
@@ -155,7 +155,6 @@ class ModelConfig:
155
limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
156
use_async_output_proc: bool = True,
157
config_format: ConfigFormat = ConfigFormat.AUTO,
158
- chat_template_text_format: str = "string",
158
hf_overrides: Optional[HfOverrides] = None,
159
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160
override_neuron_config: Optional[Dict[str, Any]] = None,
@@ -216,7 +215,6 @@ class ModelConfig:
215
self.model, revision)
216
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
217
self.use_async_output_proc = use_async_output_proc
219
- self.chat_template_text_format = chat_template_text_format
218
self.mm_processor_kwargs = mm_processor_kwargs
219
220
# Set enforce_eager to False if the value is unset.
vllm/engine/arg_utils.py
-10
@@ -90,7 +90,6 @@ class EngineArgs:
90
task: TaskOption = "auto"
91
skip_tokenizer_init: bool = False
92
tokenizer_mode: str = 'auto'
93
- chat_template_text_format: str = 'string'
93
trust_remote_code: bool = False
94
allowed_local_media_path: str = ""
95
download_dir: Optional[str] = None
@@ -258,14 +257,6 @@ class EngineArgs:
257
'fast tokenizer if available.\n* "slow" will '
258
'always use the slow tokenizer. \n* '
259
'"mistral" will always use the `mistral_common` tokenizer.')
261
- parser.add_argument(
262
- '--chat-template-text-format',
263
- type=str,
264
- default=EngineArgs.chat_template_text_format,
265
- choices=['string', 'openai'],
266
- help='The format to render text content within a chat template. '
267
- '"string" will keep the content field as a string whereas '
268
- '"openai" will parse content in the current OpenAI format.')
260
parser.add_argument('--trust-remote-code',
261
action='store_true',
262
help='Trust remote code from huggingface.')
@@ -894,7 +885,6 @@ class EngineArgs:
885
# We know this is not None because we set it in __post_init__
886
tokenizer=cast(str, self.tokenizer),
887
tokenizer_mode=self.tokenizer_mode,
897
- chat_template_text_format=self.chat_template_text_format,
888
trust_remote_code=self.trust_remote_code,
889
allowed_local_media_path=self.allowed_local_media_path,
890
dtype=self.dtype,
vllm/engine/llm_engine.py
+1
-3
@@ -262,8 +262,7 @@ class LLMEngine:
262
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
263
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
264
"use_async_output_proc=%s, use_cached_outputs=%s, "
265
- "chat_template_text_format=%s, mm_processor_kwargs=%s, "
266
- "pooler_config=%r)",
265
+ "mm_processor_kwargs=%s, pooler_config=%r)",
266
VLLM_VERSION,
267
model_config.model,
268
speculative_config,
@@ -296,7 +295,6 @@ class LLMEngine:
295
cache_config.enable_prefix_caching,
296
model_config.use_async_output_proc,
297
use_cached_outputs,
299
- model_config.chat_template_text_format,
298
model_config.mm_processor_kwargs,
299
model_config.pooler_config,
300
)
vllm/entrypoints/chat_utils.py
+223
-23
@@ -2,12 +2,14 @@ import asyncio
2
import codecs
3
import json
4
from abc import ABC, abstractmethod
5
-from collections import defaultdict
5
+from collections import defaultdict, deque
6
from functools import lru_cache, partial
7
from pathlib import Path
8
from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9
Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
10
11
+import jinja2.nodes
12
+import transformers.utils.chat_template_utils as hf_chat_utils
13
# yapf conflicts with isort for this block
14
# yapf: disable
15
from openai.types.chat import (ChatCompletionAssistantMessageParam,
@@ -153,6 +155,199 @@ class ConversationMessage(TypedDict, total=False):
155
"""The tool calls generated by the model, such as function calls."""
156
157
158
+# Passed in by user
159
+ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]
160
+
161
+# Used internally
162
+_ChatTemplateContentFormat = Literal["string", "openai"]
163
+
164
+
165
+def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
166
+ if isinstance(node, jinja2.nodes.Name):
167
+ return node.ctx == "load" and node.name == varname
168
+
169
+ return False
170
+
171
+
172
+def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
173
+ if isinstance(node, jinja2.nodes.Getitem):
174
+ return (_is_var_access(node.node, varname)
175
+ and isinstance(node.arg, jinja2.nodes.Const)
176
+ and node.arg.value == key)
177
+
178
+ if isinstance(node, jinja2.nodes.Getattr):
179
+ return _is_var_access(node.node, varname) and node.attr == key
180
+
181
+ return False
182
+
183
+
184
+def _is_var_or_elems_access(
185
+ node: jinja2.nodes.Node,
186
+ varname: str,
187
+ key: Optional[str] = None,
188
+) -> bool:
189
+ if isinstance(node, jinja2.nodes.Filter):
190
+ return (node.node is not None
191
+ and _is_var_or_elems_access(node.node, varname, key))
192
+ if isinstance(node, jinja2.nodes.Test):
193
+ return _is_var_or_elems_access(node.node, varname, key)
194
+
195
+ if (isinstance(node, jinja2.nodes.Getitem)
196
+ and isinstance(node.arg, jinja2.nodes.Slice)):
197
+ return _is_var_or_elems_access(node.node, varname, key)
198
+
199
+ # yapf: disable
200
+ return (
201
+ _is_attr_access(node, varname, key) if key
202
+ else _is_var_access(node, varname)
203
+ ) # yapf: enable
204
+
205
+
206
+def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
207
+ # Global variable that is implicitly defined at the root
208
+ yield root, varname
209
+
210
+ # Iterative BFS
211
+ related_varnames = deque([varname])
212
+ while related_varnames:
213
+ related_varname = related_varnames.popleft()
214
+
215
+ for assign_ast in root.find_all(jinja2.nodes.Assign):
216
+ lhs = assign_ast.target
217
+ rhs = assign_ast.node
218
+
219
+ if _is_var_or_elems_access(rhs, related_varname):
220
+ assert isinstance(lhs, jinja2.nodes.Name)
221
+ yield assign_ast, lhs.name
222
+
223
+ # Avoid infinite looping for self-assignment
224
+ if lhs.name != related_varname:
225
+ related_varnames.append(lhs.name)
226
+
227
+
228
+# NOTE: The proper way to handle this is to build a CFG so that we can handle
229
+# the scope in which each variable is defined, but that is too complicated
230
+def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
231
+ messages_varnames = [
232
+ varname
233
+ for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
234
+ ]
235
+
236
+ # Search for {%- for message in messages -%} loops
237
+ for loop_ast in root.find_all(jinja2.nodes.For):
238
+ loop_iter = loop_ast.iter
239
+ loop_target = loop_ast.target
240
+
241
+ for varname in messages_varnames:
242
+ if _is_var_or_elems_access(loop_iter, varname):
243
+ assert isinstance(loop_target, jinja2.nodes.Name)
244
+ yield loop_ast, loop_target.name
245
+ break
246
+
247
+
248
+def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
249
+ message_varnames = [
250
+ varname for _, varname in _iter_nodes_assign_messages_item(root)
251
+ ]
252
+
253
+ # Search for {%- for content in message['content'] -%} loops
254
+ for loop_ast in root.find_all(jinja2.nodes.For):
255
+ loop_iter = loop_ast.iter
256
+ loop_target = loop_ast.target
257
+
258
+ for varname in message_varnames:
259
+ if _is_var_or_elems_access(loop_iter, varname, "content"):
260
+ assert isinstance(loop_target, jinja2.nodes.Name)
261
+ yield loop_ast, loop_target.name
262
+ break
263
+
264
+
265
+def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
266
+ try:
267
+ jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
268
+ return jinja_compiled.environment.parse(chat_template)
269
+ except Exception:
270
+ logger.exception("Error when compiling Jinja template")
271
+ return None
272
+
273
+
274
+def _detect_content_format(
275
+ chat_template: str,
276
+ *,
277
+ default: _ChatTemplateContentFormat,
278
+) -> _ChatTemplateContentFormat:
279
+ jinja_ast = _try_extract_ast(chat_template)
280
+ if jinja_ast is None:
281
+ return default
282
+
283
+ try:
284
+ next(_iter_nodes_assign_content_item(jinja_ast))
285
+ except StopIteration:
286
+ return "string"
287
+ except Exception:
288
+ logger.exception("Error when parsing AST of Jinja template")
289
+ return default
290
+ else:
291
+ return "openai"
292
+
293
+
294
+def _resolve_chat_template_content_format(
295
+ chat_template: Optional[str],
296
+ given_format: ChatTemplateContentFormatOption,
297
+ tokenizer: AnyTokenizer,
298
+) -> _ChatTemplateContentFormat:
299
+ if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
300
+ tokenizer_chat_template = tokenizer.chat_template
301
+ else:
302
+ tokenizer_chat_template = None
303
+
304
+ jinja_text: Optional[str]
305
+ if isinstance(tokenizer_chat_template, str) and chat_template is None:
306
+ jinja_text = tokenizer_chat_template
307
+ elif (isinstance(tokenizer_chat_template, dict)
308
+ and chat_template in tokenizer_chat_template):
309
+ jinja_text = tokenizer_chat_template[chat_template]
310
+ else:
311
+ jinja_text = load_chat_template(chat_template, is_literal=True)
312
+
313
+ detected_format = ("string" if jinja_text is None else
314
+ _detect_content_format(jinja_text, default="string"))
315
+
316
+ return detected_format if given_format == "auto" else given_format
317
+
318
+
319
+@lru_cache
320
+def resolve_chat_template_content_format(
321
+ chat_template: Optional[str],
322
+ given_format: ChatTemplateContentFormatOption,
323
+ tokenizer: AnyTokenizer,
324
+) -> _ChatTemplateContentFormat:
325
+ detected_format = _resolve_chat_template_content_format(
326
+ chat_template,
327
+ given_format,
328
+ tokenizer,
329
+ )
330
+
331
+ logger.info(
332
+ "Detected the chat template content format to be '%s'. "
333
+ "You can set `--chat-template-content-format` to override this.",
334
+ detected_format,
335
+ )
336
+
337
+ if given_format != "auto" and given_format != detected_format:
338
+ logger.warning(
339
+ "You specified `--chat-template-content-format %s` "
340
+ "which is different from the detected format '%s'. "
341
+ "If our automatic detection is incorrect, please consider "
342
+ "opening a GitHub issue so that we can improve it: "
343
+ "https://github.com/vllm-project/vllm/issues/new/choose",
344
+ given_format,
345
+ detected_format,
346
+ )
347
+
348
+ return detected_format
349
+
350
+
351
ModalityStr = Literal["image", "audio", "video"]
352
_T = TypeVar("_T")
353
@@ -407,12 +602,23 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):
602
603
604
def load_chat_template(
410
- chat_template: Optional[Union[Path, str]]) -> Optional[str]:
605
+ chat_template: Optional[Union[Path, str]],
606
+ *,
607
+ is_literal: bool = False,
608
+) -> Optional[str]:
609
if chat_template is None:
610
return None
611
+
612
+ if is_literal:
613
+ if isinstance(chat_template, Path):
614
+ raise TypeError("chat_template is expected to be read directly "
615
+ "from its value")
616
+
617
+ return codecs.decode(chat_template, "unicode_escape")
618
+
619
try:
620
with open(chat_template) as f:
415
- resolved_chat_template = f.read()
621
+ return f.read()
622
except OSError as e:
623
if isinstance(chat_template, Path):
624
raise
@@ -426,10 +632,7 @@ def load_chat_template(
632
633
# If opening a file fails, set chat template to be args to
634
# ensure we decode so our escape are interpreted correctly
429
- resolved_chat_template = codecs.decode(chat_template, "unicode_escape")
430
-
431
- logger.info("Using supplied chat template:\n%s", resolved_chat_template)
432
- return resolved_chat_template
635
+ return load_chat_template(chat_template, is_literal=True)
636
637
638
# TODO: Let user specify how to insert multimodal tokens into prompt
@@ -464,7 +667,6 @@ _ImageParser = partial(cast, ChatCompletionContentPartImageParam)
667
_AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
668
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
669
_VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
467
-MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
670
671
# Define a mapping from part types to their corresponding parsing functions.
672
MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
@@ -542,18 +744,12 @@ def _parse_chat_message_content_parts(
744
role: str,
745
parts: Iterable[ChatCompletionContentPartParam],
746
mm_tracker: BaseMultiModalItemTracker,
545
- chat_template_text_format: str,
747
+ *,
748
+ wrap_dicts: bool,
749
) -> List[ConversationMessage]:
750
content: List[Union[str, Dict[str, str]]] = []
751
752
mm_parser = mm_tracker.create_parser()
550
- model_config = mm_tracker.model_config
551
-
552
- wrap_dicts = (chat_template_text_format == "openai"
553
- or (model_config.task == "embedding"
554
- and model_config.is_multimodal_model)
555
- or (model_config.hf_config.model_type
556
- in MODEL_KEEP_MULTI_MODAL_CONTENT))
753
754
for part in parts:
755
parse_res = _parse_chat_message_content_part(
@@ -578,9 +774,11 @@ def _parse_chat_message_content_parts(
774
775
776
def _parse_chat_message_content_part(
581
- part: ChatCompletionContentPartParam,
582
- mm_parser: BaseMultiModalContentParser,
583
- wrap_dicts: bool) -> Optional[Union[str, Dict[str, str]]]:
777
+ part: ChatCompletionContentPartParam,
778
+ mm_parser: BaseMultiModalContentParser,
779
+ *,
780
+ wrap_dicts: bool,
781
+) -> Optional[Union[str, Dict[str, str]]]:
782
"""Parses a single part of a conversation. If wrap_dicts is True,
783
structured dictionary pieces for texts and images will be
784
wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
@@ -629,7 +827,7 @@ _ToolParser = partial(cast, ChatCompletionToolMessageParam)
827
def _parse_chat_message_content(
828
message: ChatCompletionMessageParam,
829
mm_tracker: BaseMultiModalItemTracker,
632
- chat_template_text_format: str,
830
+ content_format: _ChatTemplateContentFormat,
831
) -> List[ConversationMessage]:
832
role = message["role"]
833
content = message.get("content")
@@ -645,7 +843,7 @@ def _parse_chat_message_content(
843
role,
844
content, # type: ignore
845
mm_tracker,
648
- chat_template_text_format,
846
+ wrap_dicts=(content_format == "openai"),
847
)
848
849
for result_msg in result:
@@ -684,6 +882,7 @@ def parse_chat_messages(
882
messages: List[ChatCompletionMessageParam],
883
model_config: ModelConfig,
884
tokenizer: AnyTokenizer,
885
+ content_format: _ChatTemplateContentFormat,
886
) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:
887
conversation: List[ConversationMessage] = []
888
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
@@ -692,7 +891,7 @@ def parse_chat_messages(
891
sub_messages = _parse_chat_message_content(
892
msg,
893
mm_tracker,
695
- model_config.chat_template_text_format,
894
+ content_format,
895
)
896
897
conversation.extend(sub_messages)
@@ -706,6 +905,7 @@ def parse_chat_messages_futures(
905
messages: List[ChatCompletionMessageParam],
906
model_config: ModelConfig,
907
tokenizer: AnyTokenizer,
908
+ content_format: _ChatTemplateContentFormat,
909
) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
910
conversation: List[ConversationMessage] = []
911
mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
@@ -714,7 +914,7 @@ def parse_chat_messages_futures(
914
sub_messages = _parse_chat_message_content(
915
msg,
916
mm_tracker,
717
- model_config.chat_template_text_format,
917
+ content_format,
918
)
919
920
conversation.extend(sub_messages)
vllm/entrypoints/llm.py
+33
-11
@@ -13,9 +13,11 @@ from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
13
TaskOption)
14
from vllm.engine.llm_engine import LLMEngine
15
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
16
+ ChatTemplateContentFormatOption,
17
apply_hf_chat_template,
18
apply_mistral_chat_template,
18
- parse_chat_messages)
19
+ parse_chat_messages,
20
+ resolve_chat_template_content_format)
21
from vllm.inputs import PromptType, TextPrompt, TokensPrompt
22
from vllm.inputs.parse import parse_and_batch_prompt
23
from vllm.logger import init_logger
@@ -523,6 +525,7 @@ class LLM:
525
use_tqdm: bool = True,
526
lora_request: Optional[LoRARequest] = None,
527
chat_template: Optional[str] = None,
528
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
529
add_generation_prompt: bool = True,
530
continue_final_message: bool = False,
531
tools: Optional[List[Dict[str, Any]]] = None,
@@ -539,9 +542,11 @@ class LLM:
542
to the OpenAI API.
543
544
Args:
542
- messages: A list of conversations or a single conversation.
543
- - Each conversation is represented as a list of messages.
544
- - Each message is a dictionary with 'role' and 'content' keys.
545
+ messages: A list of conversations or a single conversation.
546
+
547
+ - Each conversation is represented as a list of messages.
548
+ - Each message is a dictionary with 'role' and 'content' keys.
549
+
550
sampling_params: The sampling parameters for text generation.
551
If None, we use the default sampling parameters. When it
552
is a single value, it is applied to every prompt. When it
@@ -551,11 +556,19 @@ class LLM:
556
lora_request: LoRA request to use for generation, if any.
557
chat_template: The template to use for structuring the chat.
558
If not provided, the model's default chat template will be used.
559
+ chat_template_content_format: The format to render message content.
560
+
561
+ - "string" will render the content as a string.
562
+ Example: ``"Who are you?"``
563
+ - "openai" will render the content as a list of dictionaries,
564
+ similar to OpenAI schema.
565
+ Example: ``[{"type": "text", "text": "Who are you?"}]``
566
+
567
add_generation_prompt: If True, adds a generation template
568
to each message.
569
continue_final_message: If True, continues the final message in
557
- the conversation instead of starting a new one. Cannot be `True`
558
- if `add_generation_prompt` is also `True`.
570
+ the conversation instead of starting a new one. Cannot be
571
+ ``True`` if ``add_generation_prompt`` is also ``True``.
572
mm_processor_kwargs: Multimodal processor kwarg overrides for this
573
chat request. Only used for offline requests.
574
@@ -576,17 +589,26 @@ class LLM:
589
cast(List[ChatCompletionMessageParam], messages)
590
]
591
592
+ tokenizer = self.get_tokenizer()
593
+ model_config = self.llm_engine.get_model_config()
594
+ resolved_content_format = resolve_chat_template_content_format(
595
+ chat_template,
596
+ chat_template_content_format,
597
+ tokenizer,
598
+ )
599
+
600
prompts: List[Union[TokensPrompt, TextPrompt]] = []
601
602
for msgs in list_of_messages:
582
- tokenizer = self.get_tokenizer()
583
- model_config = self.llm_engine.get_model_config()
584
-
603
# NOTE: _parse_chat_message_content_parts() currently doesn't
604
# handle mm_processor_kwargs, since there is no implementation in
605
# the chat message parsing for it.
606
conversation, mm_data = parse_chat_messages(
589
- msgs, model_config, tokenizer)
607
+ msgs,
608
+ model_config,
609
+ tokenizer,
610
+ content_format=resolved_content_format,
611
+ )
612
613
prompt_data: Union[str, List[int]]
614
if isinstance(tokenizer, MistralTokenizer):
@@ -737,7 +759,7 @@ class LLM:
759
generation, if any.
760
761
Returns:
740
- A list of `EmbeddingRequestOutput` objects containing the
762
+ A list of ``EmbeddingRequestOutput`` objects containing the
763
generated embeddings in the same order as the input prompts.
764
765
Note:
vllm/entrypoints/openai/api_server.py
+10
-3
@@ -29,6 +29,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
29
from vllm.engine.multiprocessing.client import MQLLMEngineClient
30
from vllm.engine.multiprocessing.engine import run_mp_engine
31
from vllm.engine.protocol import EngineClient
32
+from vllm.entrypoints.chat_utils import load_chat_template
33
from vllm.entrypoints.launcher import serve_http
34
from vllm.entrypoints.logger import RequestLogger
35
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
@@ -529,6 +530,9 @@ def init_app_state(
530
state.engine_client = engine_client
531
state.log_stats = not args.disable_log_stats
532
533
+ resolved_chat_template = load_chat_template(args.chat_template)
534
+ logger.info("Using supplied chat template:\n%s", resolved_chat_template)
535
+
536
state.openai_serving_chat = OpenAIServingChat(
537
engine_client,
538
model_config,
@@ -537,7 +541,8 @@ def init_app_state(
541
lora_modules=args.lora_modules,
542
prompt_adapters=args.prompt_adapters,
543
request_logger=request_logger,
540
- chat_template=args.chat_template,
544
+ chat_template=resolved_chat_template,
545
+ chat_template_content_format=args.chat_template_content_format,
546
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
547
enable_auto_tools=args.enable_auto_tool_choice,
548
tool_parser=args.tool_call_parser,
@@ -557,7 +562,8 @@ def init_app_state(
562
model_config,
563
base_model_paths,
564
request_logger=request_logger,
560
- chat_template=args.chat_template,
565
+ chat_template=resolved_chat_template,
566
+ chat_template_content_format=args.chat_template_content_format,
567
) if model_config.task == "embedding" else None
568
state.openai_serving_tokenization = OpenAIServingTokenization(
569
engine_client,
@@ -565,7 +571,8 @@ def init_app_state(
571
base_model_paths,
572
lora_modules=args.lora_modules,
573
request_logger=request_logger,
568
- chat_template=args.chat_template,
574
+ chat_template=resolved_chat_template,
575
+ chat_template_content_format=args.chat_template_content_format,
576
)
577
578
vllm/entrypoints/openai/cli_args.py
+15
-2
@@ -7,10 +7,11 @@ purposes.
7
import argparse
8
import json
9
import ssl
10
-from typing import List, Optional, Sequence, Union
10
+from typing import List, Optional, Sequence, Union, get_args
11
12
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13
-from vllm.entrypoints.chat_utils import validate_chat_template
13
+from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14
+ validate_chat_template)
15
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
16
PromptAdapterPath)
17
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -132,6 +133,18 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
133
help="The file path to the chat template, "
134
"or the template in single-line form "
135
"for the specified model")
136
+ parser.add_argument(
137
+ '--chat-template-content-format',
138
+ type=str,
139
+ default="auto",
140
+ choices=get_args(ChatTemplateContentFormatOption),
141
+ help='The format to render message content within a chat template.'
142
+ '\n\n'
143
+ '* "string" will render the content as a string. '
144
+ 'Example: "Hello World"\n'
145
+ '* "openai" will render the content as a list of dictionaries, '
146
+ 'similar to OpenAI schema. '
147
+ 'Example: [{"type": "text", "text": "Hello world!"}]')
148
parser.add_argument("--response-role",
149
type=nullable_str,
150
default="assistant",
vllm/entrypoints/openai/protocol.py
+45
-26
@@ -5,9 +5,8 @@ from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
6
7
import torch
8
-from openai.types.chat import ChatCompletionContentPartParam
8
from pydantic import BaseModel, ConfigDict, Field, model_validator
10
-from typing_extensions import Annotated, Required, TypedDict
9
+from typing_extensions import Annotated
10
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
from vllm.pooling_params import PoolingParams
@@ -35,26 +34,6 @@ assert _LONG_INFO.min == _MOCK_LONG_INFO.min
34
assert _LONG_INFO.max == _MOCK_LONG_INFO.max
35
36
38
-class CustomChatCompletionMessageParam(TypedDict, total=False):
39
- """Enables custom roles in the Chat Completion API."""
40
- role: Required[str]
41
- """The role of the message's author."""
42
-
43
- content: Union[str, List[ChatCompletionContentPartParam]]
44
- """The contents of the message."""
45
-
46
- name: str
47
- """An optional name for the participant.
48
-
49
- Provides the model information to differentiate between participants of the
50
- same role.
51
- """
52
-
53
- tool_call_id: Optional[str]
54
-
55
- tool_calls: Optional[List[dict]]
56
-
57
-
37
class OpenAIBaseModel(BaseModel):
38
# OpenAI API does not allow extra fields
39
model_config = ConfigDict(extra="forbid")
@@ -1054,16 +1033,56 @@ class TokenizeCompletionRequest(OpenAIBaseModel):
1033
model: str
1034
prompt: str
1035
1057
- add_special_tokens: bool = Field(default=True)
1036
+ add_special_tokens: bool = Field(
1037
+ default=True,
1038
+ description=(
1039
+ "If true (the default), special tokens (e.g. BOS) will be added to "
1040
+ "the prompt."),
1041
+ )
1042
1043
1044
class TokenizeChatRequest(OpenAIBaseModel):
1045
model: str
1046
messages: List[ChatCompletionMessageParam]
1047
1064
- add_generation_prompt: bool = Field(default=True)
1065
- continue_final_message: bool = Field(default=False)
1066
- add_special_tokens: bool = Field(default=False)
1048
+ add_generation_prompt: bool = Field(
1049
+ default=True,
1050
+ description=
1051
+ ("If true, the generation prompt will be added to the chat template. "
1052
+ "This is a parameter used by chat template in tokenizer config of the "
1053
+ "model."),
1054
+ )
1055
+ continue_final_message: bool = Field(
1056
+ default=False,
1057
+ description=
1058
+ ("If this is set, the chat will be formatted so that the final "
1059
+ "message in the chat is open-ended, without any EOS tokens. The "
1060
+ "model will continue this message rather than starting a new one. "
1061
+ "This allows you to \"prefill\" part of the model's response for it. "
1062
+ "Cannot be used at the same time as `add_generation_prompt`."),
1063
+ )
1064
+ add_special_tokens: bool = Field(
1065
+ default=False,
1066
+ description=(
1067
+ "If true, special tokens (e.g. BOS) will be added to the prompt "
1068
+ "on top of what is added by the chat template. "
1069
+ "For most models, the chat template takes care of adding the "
1070
+ "special tokens so this should be set to false (as is the "
1071
+ "default)."),
1072
+ )
1073
+ chat_template: Optional[str] = Field(
1074
+ default=None,
1075
+ description=(
1076
+ "A Jinja template to use for this conversion. "
1077
+ "As of transformers v4.44, default chat template is no longer "
1078
+ "allowed, so you must provide a chat template if the tokenizer "
1079
+ "does not define one."),
1080
+ )
1081
+ chat_template_kwargs: Optional[Dict[str, Any]] = Field(
1082
+ default=None,
1083
+ description=("Additional kwargs to pass to the template renderer. "
1084
+ "Will be accessible by the chat template."),
1085
+ )
1086
1087
@model_validator(mode="before")
1088
@classmethod
vllm/entrypoints/openai/run_batch.py
+2
@@ -222,6 +222,7 @@ async def main(args):
222
prompt_adapters=None,
223
request_logger=request_logger,
224
chat_template=None,
225
+ chat_template_content_format="auto",
226
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
227
) if model_config.task == "generate" else None
228
openai_serving_embedding = OpenAIServingEmbedding(
@@ -230,6 +231,7 @@ async def main(args):
231
base_model_paths,
232
request_logger=request_logger,
233
chat_template=None,
234
+ chat_template_content_format="auto",
235
) if model_config.task == "embedding" else None
236
237
tracker = BatchProgressTracker()
vllm/entrypoints/openai/serving_chat.py
+23
-17
@@ -10,7 +10,8 @@ from fastapi import Request
10
11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
-from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
13
+from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14
+ ConversationMessage)
15
from vllm.entrypoints.logger import RequestLogger
16
from vllm.entrypoints.openai.protocol import (
17
ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -38,20 +39,23 @@ logger = init_logger(__name__)
39
40
class OpenAIServingChat(OpenAIServing):
41
41
- def __init__(self,
42
- engine_client: EngineClient,
43
- model_config: ModelConfig,
44
- base_model_paths: List[BaseModelPath],
45
- response_role: str,
46
- *,
47
- lora_modules: Optional[List[LoRAModulePath]],
48
- prompt_adapters: Optional[List[PromptAdapterPath]],
49
- request_logger: Optional[RequestLogger],
50
- chat_template: Optional[str],
51
- return_tokens_as_token_ids: bool = False,
52
- enable_auto_tools: bool = False,
53
- tool_parser: Optional[str] = None,
54
- enable_prompt_tokens_details: bool = False):
42
+ def __init__(
43
+ self,
44
+ engine_client: EngineClient,
45
+ model_config: ModelConfig,
46
+ base_model_paths: List[BaseModelPath],
47
+ response_role: str,
48
+ *,
49
+ lora_modules: Optional[List[LoRAModulePath]],
50
+ prompt_adapters: Optional[List[PromptAdapterPath]],
51
+ request_logger: Optional[RequestLogger],
52
+ chat_template: Optional[str],
53
+ chat_template_content_format: ChatTemplateContentFormatOption,
54
+ return_tokens_as_token_ids: bool = False,
55
+ enable_auto_tools: bool = False,
56
+ tool_parser: Optional[str] = None,
57
+ enable_prompt_tokens_details: bool = False,
58
+ ) -> None:
59
super().__init__(engine_client=engine_client,
60
model_config=model_config,
61
base_model_paths=base_model_paths,
@@ -61,8 +65,8 @@ class OpenAIServingChat(OpenAIServing):
65
return_tokens_as_token_ids=return_tokens_as_token_ids)
66
67
self.response_role = response_role
64
- self.use_tool_use_model_template = False
65
- self.chat_template = load_chat_template(chat_template)
68
+ self.chat_template = chat_template
69
+ self.chat_template_content_format: Final = chat_template_content_format
70
71
# set up tool use
72
self.enable_auto_tools: bool = enable_auto_tools
@@ -120,6 +124,7 @@ class OpenAIServingChat(OpenAIServing):
124
) = self._maybe_get_adapters(request)
125
126
tokenizer = await self.engine_client.get_tokenizer(lora_request)
127
+
128
tool_parser = self.tool_parser
129
130
# validation for OpenAI tools
@@ -157,6 +162,7 @@ class OpenAIServingChat(OpenAIServing):
162
tokenizer,
163
request.messages,
164
chat_template=request.chat_template or self.chat_template,
165
+ chat_template_content_format=self.chat_template_content_format,
166
add_generation_prompt=request.add_generation_prompt,
167
continue_final_message=request.continue_final_message,
168
tool_dicts=tool_dicts,
vllm/entrypoints/openai/serving_embedding.py
+8
-4
@@ -1,7 +1,7 @@
1
import asyncio
2
import base64
3
import time
4
-from typing import AsyncGenerator, List, Literal, Optional, Union, cast
4
+from typing import AsyncGenerator, Final, List, Literal, Optional, Union, cast
5
6
import numpy as np
7
from fastapi import Request
@@ -9,7 +9,7 @@ from typing_extensions import assert_never
9
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import EngineClient
12
-from vllm.entrypoints.chat_utils import load_chat_template
12
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13
from vllm.entrypoints.logger import RequestLogger
14
from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
15
EmbeddingRequest,
@@ -77,7 +77,8 @@ class OpenAIServingEmbedding(OpenAIServing):
77
*,
78
request_logger: Optional[RequestLogger],
79
chat_template: Optional[str],
80
- ):
80
+ chat_template_content_format: ChatTemplateContentFormatOption,
81
+ ) -> None:
82
super().__init__(engine_client=engine_client,
83
model_config=model_config,
84
base_model_paths=base_model_paths,
@@ -85,7 +86,8 @@ class OpenAIServingEmbedding(OpenAIServing):
86
prompt_adapters=None,
87
request_logger=request_logger)
88
88
- self.chat_template = load_chat_template(chat_template)
89
+ self.chat_template = chat_template
90
+ self.chat_template_content_format: Final = chat_template_content_format
91
92
async def create_embedding(
93
self,
@@ -144,6 +146,8 @@ class OpenAIServingEmbedding(OpenAIServing):
146
tokenizer,
147
request.messages,
148
chat_template=request.chat_template or self.chat_template,
149
+ chat_template_content_format=self.
150
+ chat_template_content_format,
151
add_generation_prompt=request.add_generation_prompt,
152
continue_final_message=request.continue_final_message,
153
truncate_prompt_tokens=truncate_prompt_tokens,
vllm/entrypoints/openai/serving_engine.py
+13
-4
@@ -11,14 +11,16 @@ from typing_extensions import Annotated
11
12
from vllm.config import ModelConfig
13
from vllm.engine.protocol import EngineClient
14
+# yapf conflicts with isort for this block
15
+# yapf: disable
16
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
17
+ ChatTemplateContentFormatOption,
18
ConversationMessage,
19
apply_hf_chat_template,
20
apply_mistral_chat_template,
18
- parse_chat_messages_futures)
21
+ parse_chat_messages_futures,
22
+ resolve_chat_template_content_format)
23
from vllm.entrypoints.logger import RequestLogger
20
-# yapf conflicts with isort for this block
21
-# yapf: disable
24
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
25
CompletionRequest,
26
DetokenizeRequest,
@@ -426,7 +428,8 @@ class OpenAIServing:
428
request: ChatLikeRequest,
429
tokenizer: AnyTokenizer,
430
messages: List[ChatCompletionMessageParam],
429
- chat_template: Optional[str] = None,
431
+ chat_template: Optional[str],
432
+ chat_template_content_format: ChatTemplateContentFormatOption,
433
add_generation_prompt: bool = True,
434
continue_final_message: bool = False,
435
tool_dicts: Optional[List[Dict[str, Any]]] = None,
@@ -437,10 +440,16 @@ class OpenAIServing:
440
add_special_tokens: bool = False,
441
) -> Tuple[List[ConversationMessage], Sequence[RequestPrompt],
442
List[TokensPrompt]]:
443
+ resolved_content_format = resolve_chat_template_content_format(
444
+ chat_template,
445
+ chat_template_content_format,
446
+ tokenizer,
447
+ )
448
conversation, mm_data_future = parse_chat_messages_futures(
449
messages,
450
self.model_config,
451
tokenizer,
452
+ content_format=resolved_content_format,
453
)
454
455
_chat_template_kwargs: Dict[str, Any] = dict(
vllm/entrypoints/openai/serving_tokenization.py
+10
-10
@@ -1,8 +1,8 @@
1
-from typing import List, Optional, Union
1
+from typing import Final, List, Optional, Union
2
3
from vllm.config import ModelConfig
4
from vllm.engine.protocol import EngineClient
5
-from vllm.entrypoints.chat_utils import load_chat_template
5
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
6
from vllm.entrypoints.logger import RequestLogger
7
# yapf conflicts with isort for this block
8
# yapf: disable
@@ -33,7 +33,8 @@ class OpenAIServingTokenization(OpenAIServing):
33
lora_modules: Optional[List[LoRAModulePath]],
34
request_logger: Optional[RequestLogger],
35
chat_template: Optional[str],
36
- ):
36
+ chat_template_content_format: ChatTemplateContentFormatOption,
37
+ ) -> None:
38
super().__init__(engine_client=engine_client,
39
model_config=model_config,
40
base_model_paths=base_model_paths,
@@ -41,12 +42,8 @@ class OpenAIServingTokenization(OpenAIServing):
42
prompt_adapters=None,
43
request_logger=request_logger)
44
44
- # If this is None we use the tokenizer's default chat template
45
- # the list of commonly-used chat template names for HF named templates
46
- hf_chat_templates: List[str] = ['default', 'tool_use']
47
- self.chat_template = chat_template \
48
- if chat_template in hf_chat_templates \
49
- else load_chat_template(chat_template)
45
+ self.chat_template = chat_template
46
+ self.chat_template_content_format: Final = chat_template_content_format
47
48
async def create_tokenize(
49
self,
@@ -75,9 +72,12 @@ class OpenAIServingTokenization(OpenAIServing):
72
request,
73
tokenizer,
74
request.messages,
78
- chat_template=self.chat_template,
75
+ chat_template=request.chat_template or self.chat_template,
76
+ chat_template_content_format=self.
77
+ chat_template_content_format,
78
add_generation_prompt=request.add_generation_prompt,
79
continue_final_message=request.continue_final_message,
80
+ chat_template_kwargs=request.chat_template_kwargs,
81
add_special_tokens=request.add_special_tokens,
82
)
83
else: