@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32e46e000f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Frontend] Automatic detection of chat content format from AST (#9919)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 16, 2024 at 13:35 UTC 32e46e000f77499f4dd7c0bed194e33856f2df24
16 files changed +788 -350
docs/source/serving/openai_compatible_server.md
+13 -5
@@ -172,12 +172,20 @@ completion = client.chat.completions.create(
172 ]
173 )
174 ```
175 -Most chat templates for LLMs expect the `content` to be a `string` but there are some newer models like
176 -`meta-llama/Llama-Guard-3-1B` that expect the content to be parsed with the new OpenAI spec. In order to choose which
177 -format the content needs to be parsed in by vLLM, please use the `--chat-template-text-format` argument to specify
178 -between `string` or `openai`. The default value is `string` and vLLM internally converts both spec formats to match
179 -this, unless explicitly specified.
175
176 +Most chat templates for LLMs expect the `content` field to be a string, but there are some newer models like
177 +`meta-llama/Llama-Guard-3-1B` that expect the content to be formatted according to the OpenAI schema in the
178 +request. vLLM provides best-effort support to detect this automatically, which is logged as a string like
179 +*"Detected the chat template content format to be..."*, and internally converts incoming requests to match
180 +the detected format, which can be one of:
181 +
182 +- `"string"`: A string.
183 + - Example: `"Hello world"`
184 +- `"openai"`: A list of dictionaries, similar to OpenAI schema.
185 + - Example: `[{"type": "text", "text": "Hello world!"}]`
186 +
187 +If the result is not what you expect, you can set the `--chat-template-content-format` CLI argument
188 +to override which format to use.
189
190 ## Command line arguments for the server
191
tests/entrypoints/openai/test_serving_chat.py
+2 -1
@@ -26,7 +26,6 @@ class MockModelConfig:
26 tokenizer = MODEL_NAME
27 trust_remote_code = False
28 tokenizer_mode = "auto"
29 - chat_template_text_format = "string"
29 max_model_len = 100
30 tokenizer_revision = None
31 multimodal_config = MultiModalConfig()
@@ -49,6 +48,7 @@ async def _async_serving_chat_init():
48 BASE_MODEL_PATHS,
49 response_role="assistant",
50 chat_template=CHAT_TEMPLATE,
51 + chat_template_content_format="auto",
52 lora_modules=None,
53 prompt_adapters=None,
54 request_logger=None)
@@ -70,6 +70,7 @@ def test_serving_chat_should_set_correct_max_tokens():
70 BASE_MODEL_PATHS,
71 response_role="assistant",
72 chat_template=CHAT_TEMPLATE,
73 + chat_template_content_format="auto",
74 lora_modules=None,
75 prompt_adapters=None,
76 request_logger=None)
tests/entrypoints/test_chat_utils.py
+390 -229
@@ -6,15 +6,24 @@ from PIL import Image
6
7 from vllm.assets.image import ImageAsset
8 from vllm.config import ModelConfig
9 -from vllm.entrypoints.chat_utils import (parse_chat_messages,
10 - parse_chat_messages_futures)
9 +from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
10 + parse_chat_messages,
11 + parse_chat_messages_futures,
12 + resolve_chat_template_content_format)
13 from vllm.entrypoints.llm import apply_hf_chat_template
14 from vllm.multimodal import MultiModalDataDict
15 from vllm.multimodal.utils import encode_image_base64
16 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
17
18 +from ..utils import VLLM_PATH
19 +
20 +EXAMPLES_DIR = VLLM_PATH / "examples"
21 +
22 PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
23 +ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_3"
24 +QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
25 MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
26 +LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
27
28
29 @pytest.fixture(scope="function")
@@ -26,7 +35,6 @@ def phi3v_model_config():
35 trust_remote_code=True,
36 dtype="bfloat16",
37 seed=0,
29 - chat_template_text_format="string",
38 limit_mm_per_prompt={
39 "image": 2,
40 })
@@ -94,19 +102,24 @@ def test_parse_chat_messages_single_image(
102 phi3v_tokenizer,
103 image_url,
104 ):
97 - conversation, mm_data = parse_chat_messages([{
98 - "role":
99 - "user",
100 - "content": [{
101 - "type": "image_url",
102 - "image_url": {
103 - "url": image_url
104 - }
105 - }, {
106 - "type": "text",
107 - "text": "What's in the image?"
108 - }]
109 - }], phi3v_model_config, phi3v_tokenizer)
105 + conversation, mm_data = parse_chat_messages(
106 + [{
107 + "role":
108 + "user",
109 + "content": [{
110 + "type": "image_url",
111 + "image_url": {
112 + "url": image_url
113 + }
114 + }, {
115 + "type": "text",
116 + "text": "What's in the image?"
117 + }]
118 + }],
119 + phi3v_model_config,
120 + phi3v_tokenizer,
121 + content_format="string",
122 + )
123
124 assert conversation == [{
125 "role": "user",
@@ -121,19 +134,24 @@ async def test_parse_chat_messages_single_image_async(
134 phi3v_tokenizer,
135 image_url,
136 ):
124 - conversation, mm_future = parse_chat_messages_futures([{
125 - "role":
126 - "user",
127 - "content": [{
128 - "type": "image_url",
129 - "image_url": {
130 - "url": image_url
131 - }
132 - }, {
133 - "type": "text",
134 - "text": "What's in the image?"
135 - }]
136 - }], phi3v_model_config, phi3v_tokenizer)
137 + conversation, mm_future = parse_chat_messages_futures(
138 + [{
139 + "role":
140 + "user",
141 + "content": [{
142 + "type": "image_url",
143 + "image_url": {
144 + "url": image_url
145 + }
146 + }, {
147 + "type": "text",
148 + "text": "What's in the image?"
149 + }]
150 + }],
151 + phi3v_model_config,
152 + phi3v_tokenizer,
153 + content_format="string",
154 + )
155
156 assert conversation == [{
157 "role": "user",
@@ -147,24 +165,29 @@ def test_parse_chat_messages_multiple_images(
165 phi3v_tokenizer,
166 image_url,
167 ):
150 - conversation, mm_data = parse_chat_messages([{
151 - "role":
152 - "user",
153 - "content": [{
154 - "type": "image_url",
155 - "image_url": {
156 - "url": image_url
157 - }
158 - }, {
159 - "type": "image_url",
160 - "image_url": {
161 - "url": image_url
162 - }
163 - }, {
164 - "type": "text",
165 - "text": "What's in these images?"
166 - }]
167 - }], phi3v_model_config, phi3v_tokenizer)
168 + conversation, mm_data = parse_chat_messages(
169 + [{
170 + "role":
171 + "user",
172 + "content": [{
173 + "type": "image_url",
174 + "image_url": {
175 + "url": image_url
176 + }
177 + }, {
178 + "type": "image_url",
179 + "image_url": {
180 + "url": image_url
181 + }
182 + }, {
183 + "type": "text",
184 + "text": "What's in these images?"
185 + }]
186 + }],
187 + phi3v_model_config,
188 + phi3v_tokenizer,
189 + content_format="string",
190 + )
191
192 assert conversation == [{
193 "role":
@@ -181,24 +204,29 @@ async def test_parse_chat_messages_multiple_images_async(
204 phi3v_tokenizer,
205 image_url,
206 ):
184 - conversation, mm_future = parse_chat_messages_futures([{
185 - "role":
186 - "user",
187 - "content": [{
188 - "type": "image_url",
189 - "image_url": {
190 - "url": image_url
191 - }
192 - }, {
193 - "type": "image_url",
194 - "image_url": {
195 - "url": image_url
196 - }
197 - }, {
198 - "type": "text",
199 - "text": "What's in these images?"
200 - }]
201 - }], phi3v_model_config, phi3v_tokenizer)
207 + conversation, mm_future = parse_chat_messages_futures(
208 + [{
209 + "role":
210 + "user",
211 + "content": [{
212 + "type": "image_url",
213 + "image_url": {
214 + "url": image_url
215 + }
216 + }, {
217 + "type": "image_url",
218 + "image_url": {
219 + "url": image_url
220 + }
221 + }, {
222 + "type": "text",
223 + "text": "What's in these images?"
224 + }]
225 + }],
226 + phi3v_model_config,
227 + phi3v_tokenizer,
228 + content_format="string",
229 + )
230
231 assert conversation == [{
232 "role":
@@ -214,27 +242,31 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
242 phi3v_tokenizer,
243 image_url,
244 ):
217 - conversation, mm_data = parse_chat_messages([{
218 - "role":
219 - "user",
220 - "content": [{
221 - "type": "image_url",
222 - "image_url": {
223 - "url": image_url
224 - }
225 - }, {
226 - "type": "image_url",
227 - "image_url": {
228 - "url": image_url
229 - }
230 - }, {
231 - "type":
232 - "text",
233 - "text":
234 - "What's in <|image_1|> and how does it compare to <|image_2|>?"
235 - }]
236 - }], phi3v_model_config, phi3v_tokenizer)
237 -
245 + conversation, mm_data = parse_chat_messages(
246 + [{
247 + "role":
248 + "user",
249 + "content": [{
250 + "type": "image_url",
251 + "image_url": {
252 + "url": image_url
253 + }
254 + }, {
255 + "type": "image_url",
256 + "image_url": {
257 + "url": image_url
258 + }
259 + }, {
260 + "type":
261 + "text",
262 + "text":
263 + "What's in <|image_1|> and how does it compare to <|image_2|>?"
264 + }]
265 + }],
266 + phi3v_model_config,
267 + phi3v_tokenizer,
268 + content_format="string",
269 + )
270 assert conversation == [{
271 "role":
272 "user",
@@ -249,26 +281,35 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
281 phi3v_tokenizer,
282 image_url,
283 ):
252 - conversation, mm_data = parse_chat_messages([{
253 - "role":
254 - "user",
255 - "content": [{
256 - "type": "image_url",
257 - "image_url": {
258 - "url": image_url
259 - }
260 - }, {
261 - "type": "image_url",
262 - "image_url": {
263 - "url": image_url
264 - }
265 - }, {
266 - "type":
267 - "text",
268 - "text":
269 - "What's in <|image_1|> and how does it compare to the other one?"
270 - }]
271 - }], phi3v_model_config, phi3v_tokenizer)
284 + conversation, mm_data = parse_chat_messages(
285 + [{
286 + "role":
287 + "user",
288 + "content": [
289 + {
290 + "type": "image_url",
291 + "image_url": {
292 + "url": image_url
293 + }
294 + },
295 + {
296 + "type": "image_url",
297 + "image_url": {
298 + "url": image_url
299 + }
300 + },
301 + {
302 + "type":
303 + "text",
304 + "text":
305 + "What's in <|image_1|> and how does it compare to the other one?" # noqa: E501
306 + }
307 + ]
308 + }],
309 + phi3v_model_config,
310 + phi3v_tokenizer,
311 + content_format="string",
312 + )
313
314 assert conversation == [{
315 "role":
@@ -285,34 +326,39 @@ def test_parse_chat_messages_multiple_images_across_messages(
326 phi3v_tokenizer,
327 image_url,
328 ):
288 - conversation, mm_data = parse_chat_messages([{
289 - "role":
290 - "user",
291 - "content": [{
292 - "type": "image_url",
293 - "image_url": {
294 - "url": image_url
295 - }
329 + conversation, mm_data = parse_chat_messages(
330 + [{
331 + "role":
332 + "user",
333 + "content": [{
334 + "type": "image_url",
335 + "image_url": {
336 + "url": image_url
337 + }
338 + }, {
339 + "type": "text",
340 + "text": "What's in this image?"
341 + }]
342 }, {
297 - "type": "text",
298 - "text": "What's in this image?"
299 - }]
300 - }, {
301 - "role": "assistant",
302 - "content": "Some stuff."
303 - }, {
304 - "role":
305 - "user",
306 - "content": [{
307 - "type": "image_url",
308 - "image_url": {
309 - "url": image_url
310 - }
343 + "role": "assistant",
344 + "content": "Some stuff."
345 }, {
312 - "type": "text",
313 - "text": "What about this one?"
314 - }]
315 - }], phi3v_model_config, phi3v_tokenizer)
346 + "role":
347 + "user",
348 + "content": [{
349 + "type": "image_url",
350 + "image_url": {
351 + "url": image_url
352 + }
353 + }, {
354 + "type": "text",
355 + "text": "What about this one?"
356 + }]
357 + }],
358 + phi3v_model_config,
359 + phi3v_tokenizer,
360 + content_format="string",
361 + )
362
363 assert conversation == [
364 {
@@ -335,7 +381,6 @@ def test_parse_chat_messages_context_text_format(
381 phi3v_model_config,
382 phi3v_tokenizer,
383 ):
338 - phi3v_model_config.chat_template_text_format = "openai"
384 conversation, mm_data = parse_chat_messages(
385 [{
386 "role": "user",
@@ -349,7 +394,11 @@ def test_parse_chat_messages_context_text_format(
394 }, {
395 "role": "user",
396 "content": "What about this one?"
352 - }], phi3v_model_config, phi3v_tokenizer)
397 + }],
398 + phi3v_model_config,
399 + phi3v_tokenizer,
400 + content_format="openai",
401 + )
402
403 assert conversation == [
404 {
@@ -389,29 +438,34 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
438 ValueError,
439 match="At most 2 image\\(s\\) may be provided in one request\\."
440 ):
392 - parse_chat_messages([{
393 - "role":
394 - "user",
395 - "content": [{
396 - "type": "image_url",
397 - "image_url": {
398 - "url": image_url
399 - }
400 - }, {
401 - "type": "image_url",
402 - "image_url": {
403 - "url": image_url
404 - }
405 - }, {
406 - "type": "image_url",
407 - "image_url": {
408 - "url": image_url
409 - }
410 - }, {
411 - "type": "text",
412 - "text": "What's in these images?"
413 - }]
414 - }], phi3v_model_config, phi3v_tokenizer)
441 + parse_chat_messages(
442 + [{
443 + "role":
444 + "user",
445 + "content": [{
446 + "type": "image_url",
447 + "image_url": {
448 + "url": image_url
449 + }
450 + }, {
451 + "type": "image_url",
452 + "image_url": {
453 + "url": image_url
454 + }
455 + }, {
456 + "type": "image_url",
457 + "image_url": {
458 + "url": image_url
459 + }
460 + }, {
461 + "type": "text",
462 + "text": "What's in these images?"
463 + }]
464 + }],
465 + phi3v_model_config,
466 + phi3v_tokenizer,
467 + content_format="string",
468 + )
469
470
471 def test_parse_chat_messages_rejects_too_many_images_across_messages(
@@ -427,39 +481,44 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
481 ValueError,
482 match="At most 2 image\\(s\\) may be provided in one request\\."
483 ):
430 - parse_chat_messages([{
431 - "role":
432 - "user",
433 - "content": [{
434 - "type": "image_url",
435 - "image_url": {
436 - "url": image_url
437 - }
484 + parse_chat_messages(
485 + [{
486 + "role":
487 + "user",
488 + "content": [{
489 + "type": "image_url",
490 + "image_url": {
491 + "url": image_url
492 + }
493 + }, {
494 + "type": "text",
495 + "text": "What's in this image?"
496 + }]
497 }, {
439 - "type": "text",
440 - "text": "What's in this image?"
441 - }]
442 - }, {
443 - "role": "assistant",
444 - "content": "Some stuff."
445 - }, {
446 - "role":
447 - "user",
448 - "content": [{
449 - "type": "image_url",
450 - "image_url": {
451 - "url": image_url
452 - }
498 + "role": "assistant",
499 + "content": "Some stuff."
500 }, {
454 - "type": "image_url",
455 - "image_url": {
456 - "url": image_url
457 - }
458 - }, {
459 - "type": "text",
460 - "text": "What about these two?"
461 - }]
462 - }], phi3v_model_config, phi3v_tokenizer)
501 + "role":
502 + "user",
503 + "content": [{
504 + "type": "image_url",
505 + "image_url": {
506 + "url": image_url
507 + }
508 + }, {
509 + "type": "image_url",
510 + "image_url": {
511 + "url": image_url
512 + }
513 + }, {
514 + "type": "text",
515 + "text": "What about these two?"
516 + }]
517 + }],
518 + phi3v_model_config,
519 + phi3v_tokenizer,
520 + content_format="string",
521 + )
522
523
524 def test_parse_chat_messages_multiple_images_uncommon_input(
@@ -467,17 +526,22 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
526 phi3v_tokenizer,
527 image_url,
528 ):
470 - conversation, mm_data = parse_chat_messages([{
471 - "role":
472 - "user",
473 - "content": [
474 - "What's in these images?", {
475 - "image_url": image_url
476 - }, {
477 - "image_url": image_url
478 - }
479 - ]
480 - }], phi3v_model_config, phi3v_tokenizer)
529 + conversation, mm_data = parse_chat_messages(
530 + [{
531 + "role":
532 + "user",
533 + "content": [
534 + "What's in these images?", {
535 + "image_url": image_url
536 + }, {
537 + "image_url": image_url
538 + }
539 + ]
540 + }],
541 + phi3v_model_config,
542 + phi3v_tokenizer,
543 + content_format="string",
544 + )
545
546 assert conversation == [{
547 "role":
@@ -495,16 +559,21 @@ def test_mllama_single_image(
559 image_url,
560 ):
561 """Ensures that a single image is parsed correctly mllama."""
498 - conversation, mm_data = parse_chat_messages([{
499 - "role":
500 - "user",
501 - "content": [{
502 - 'type': 'text',
503 - 'text': 'The content of this image is:'
504 - }, {
505 - "image_url": image_url
506 - }]
507 - }], mllama_model_config, mllama_tokenizer)
562 + conversation, mm_data = parse_chat_messages(
563 + [{
564 + "role":
565 + "user",
566 + "content": [{
567 + 'type': 'text',
568 + 'text': 'The content of this image is:'
569 + }, {
570 + "image_url": image_url
571 + }]
572 + }],
573 + mllama_model_config,
574 + mllama_tokenizer,
575 + content_format="openai",
576 + )
577 _assert_mm_data_is_image_input(mm_data, 1)
578 assert conversation == [{
579 'role':
@@ -524,26 +593,31 @@ def test_mllama_interleaved_images(
593 image_url,
594 ):
595 """Ensures that multiple image are parsed as interleaved dicts."""
527 - conversation, mm_data = parse_chat_messages([{
528 - "role":
529 - "user",
530 - "content": [
531 - {
532 - 'type': 'text',
533 - 'text': 'The content of the first image is:'
534 - },
535 - {
536 - "image_url": image_url
537 - },
538 - {
539 - 'type': 'text',
540 - 'text': 'The content of the second image is:'
541 - },
542 - {
543 - "image_url": image_url
544 - },
545 - ]
546 - }], mllama_model_config, mllama_tokenizer)
596 + conversation, mm_data = parse_chat_messages(
597 + [{
598 + "role":
599 + "user",
600 + "content": [
601 + {
602 + 'type': 'text',
603 + 'text': 'The content of the first image is:'
604 + },
605 + {
606 + "image_url": image_url
607 + },
608 + {
609 + 'type': 'text',
610 + 'text': 'The content of the second image is:'
611 + },
612 + {
613 + "image_url": image_url
614 + },
615 + ]
616 + }],
617 + mllama_model_config,
618 + mllama_tokenizer,
619 + content_format="openai",
620 + )
621 _assert_mm_data_is_image_input(mm_data, 2)
622 assert conversation == [{
623 'role':
@@ -626,6 +700,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
700 vllm_conversation,
701 model_config,
702 tokenizer_group,
703 + content_format="openai",
704 )
705
706 vllm_result = apply_hf_chat_template(
@@ -636,3 +711,89 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
711 )
712
713 assert hf_result == vllm_result
714 +
715 +
716 +# yapf: disable
717 +@pytest.mark.parametrize(
718 + ("model", "expected_format"),
719 + [(PHI3V_MODEL_ID, "string"),
720 + (QWEN2VL_MODEL_ID, "openai"),
721 + (ULTRAVOX_MODEL_ID, "string"),
722 + (MLLAMA_MODEL_ID, "openai"),
723 + (LLAMA_GUARD_MODEL_ID, "openai")],
724 +)
725 +# yapf: enable
726 +def test_resolve_content_format_hf_defined(model, expected_format):
727 + tokenizer_group = TokenizerGroup(
728 + model,
729 + enable_lora=False,
730 + max_num_seqs=5,
731 + max_input_length=None,
732 + )
733 + tokenizer = tokenizer_group.tokenizer
734 +
735 + chat_template = tokenizer.chat_template
736 + assert isinstance(chat_template, str)
737 +
738 + print("[TEXT]")
739 + print(chat_template)
740 + print("[AST]")
741 + print(_try_extract_ast(chat_template))
742 +
743 + resolved_format = resolve_chat_template_content_format(
744 + None, # Test detecting the tokenizer's chat_template
745 + "auto",
746 + tokenizer,
747 + )
748 +
749 + assert resolved_format == expected_format
750 +
751 +
752 +# yapf: disable
753 +@pytest.mark.parametrize(
754 + ("template_path", "expected_format"),
755 + [("template_alpaca.jinja", "string"),
756 + ("template_baichuan.jinja", "string"),
757 + ("template_blip2.jinja", "string"),
758 + ("template_chatglm.jinja", "string"),
759 + ("template_chatglm2.jinja", "string"),
760 + ("template_chatml.jinja", "string"),
761 + ("template_falcon_180b.jinja", "string"),
762 + ("template_falcon.jinja", "string"),
763 + ("template_inkbot.jinja", "string"),
764 + ("template_llava.jinja", "string"),
765 + ("template_vlm2vec.jinja", "openai"),
766 + ("tool_chat_template_granite_20b_fc.jinja", "string"),
767 + ("tool_chat_template_hermes.jinja", "string"),
768 + ("tool_chat_template_internlm2_tool.jinja", "string"),
769 + ("tool_chat_template_llama3.1_json.jinja", "string"),
770 + ("tool_chat_template_llama3.2_json.jinja", "string"),
771 + ("tool_chat_template_mistral_parallel.jinja", "string"),
772 + ("tool_chat_template_mistral.jinja", "string")],
773 +)
774 +# yapf: enable
775 +def test_resolve_content_format_examples(template_path, expected_format):
776 + tokenizer_group = TokenizerGroup(
777 + PHI3V_MODEL_ID,
778 + enable_lora=False,
779 + max_num_seqs=5,
780 + max_input_length=None,
781 + )
782 + dummy_tokenizer = tokenizer_group.tokenizer
783 + dummy_tokenizer.chat_template = None
784 +
785 + chat_template = load_chat_template(EXAMPLES_DIR / template_path)
786 + assert isinstance(chat_template, str)
787 +
788 + print("[TEXT]")
789 + print(chat_template)
790 + print("[AST]")
791 + print(_try_extract_ast(chat_template))
792 +
793 + resolved_format = resolve_chat_template_content_format(
794 + chat_template,
795 + "auto",
796 + dummy_tokenizer,
797 + )
798 +
799 + assert resolved_format == expected_format
vllm/config.py
-2
@@ -155,7 +155,6 @@ class ModelConfig:
155 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
156 use_async_output_proc: bool = True,
157 config_format: ConfigFormat = ConfigFormat.AUTO,
158 - chat_template_text_format: str = "string",
158 hf_overrides: Optional[HfOverrides] = None,
159 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160 override_neuron_config: Optional[Dict[str, Any]] = None,
@@ -216,7 +215,6 @@ class ModelConfig:
215 self.model, revision)
216 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
217 self.use_async_output_proc = use_async_output_proc
219 - self.chat_template_text_format = chat_template_text_format
218 self.mm_processor_kwargs = mm_processor_kwargs
219
220 # Set enforce_eager to False if the value is unset.
vllm/engine/arg_utils.py
-10
@@ -90,7 +90,6 @@ class EngineArgs:
90 task: TaskOption = "auto"
91 skip_tokenizer_init: bool = False
92 tokenizer_mode: str = 'auto'
93 - chat_template_text_format: str = 'string'
93 trust_remote_code: bool = False
94 allowed_local_media_path: str = ""
95 download_dir: Optional[str] = None
@@ -258,14 +257,6 @@ class EngineArgs:
257 'fast tokenizer if available.\n* "slow" will '
258 'always use the slow tokenizer. \n* '
259 '"mistral" will always use the `mistral_common` tokenizer.')
261 - parser.add_argument(
262 - '--chat-template-text-format',
263 - type=str,
264 - default=EngineArgs.chat_template_text_format,
265 - choices=['string', 'openai'],
266 - help='The format to render text content within a chat template. '
267 - '"string" will keep the content field as a string whereas '
268 - '"openai" will parse content in the current OpenAI format.')
260 parser.add_argument('--trust-remote-code',
261 action='store_true',
262 help='Trust remote code from huggingface.')
@@ -894,7 +885,6 @@ class EngineArgs:
885 # We know this is not None because we set it in __post_init__
886 tokenizer=cast(str, self.tokenizer),
887 tokenizer_mode=self.tokenizer_mode,
897 - chat_template_text_format=self.chat_template_text_format,
888 trust_remote_code=self.trust_remote_code,
889 allowed_local_media_path=self.allowed_local_media_path,
890 dtype=self.dtype,
vllm/engine/llm_engine.py
+1 -3
@@ -262,8 +262,7 @@ class LLMEngine:
262 "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
263 "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
264 "use_async_output_proc=%s, use_cached_outputs=%s, "
265 - "chat_template_text_format=%s, mm_processor_kwargs=%s, "
266 - "pooler_config=%r)",
265 + "mm_processor_kwargs=%s, pooler_config=%r)",
266 VLLM_VERSION,
267 model_config.model,
268 speculative_config,
@@ -296,7 +295,6 @@ class LLMEngine:
295 cache_config.enable_prefix_caching,
296 model_config.use_async_output_proc,
297 use_cached_outputs,
299 - model_config.chat_template_text_format,
298 model_config.mm_processor_kwargs,
299 model_config.pooler_config,
300 )
vllm/entrypoints/chat_utils.py
+223 -23
@@ -2,12 +2,14 @@ import asyncio
2 import codecs
3 import json
4 from abc import ABC, abstractmethod
5 -from collections import defaultdict
5 +from collections import defaultdict, deque
6 from functools import lru_cache, partial
7 from pathlib import Path
8 from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9 Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
10
11 +import jinja2.nodes
12 +import transformers.utils.chat_template_utils as hf_chat_utils
13 # yapf conflicts with isort for this block
14 # yapf: disable
15 from openai.types.chat import (ChatCompletionAssistantMessageParam,
@@ -153,6 +155,199 @@ class ConversationMessage(TypedDict, total=False):
155 """The tool calls generated by the model, such as function calls."""
156
157
158 +# Passed in by user
159 +ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]
160 +
161 +# Used internally
162 +_ChatTemplateContentFormat = Literal["string", "openai"]
163 +
164 +
165 +def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
166 + if isinstance(node, jinja2.nodes.Name):
167 + return node.ctx == "load" and node.name == varname
168 +
169 + return False
170 +
171 +
172 +def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
173 + if isinstance(node, jinja2.nodes.Getitem):
174 + return (_is_var_access(node.node, varname)
175 + and isinstance(node.arg, jinja2.nodes.Const)
176 + and node.arg.value == key)
177 +
178 + if isinstance(node, jinja2.nodes.Getattr):
179 + return _is_var_access(node.node, varname) and node.attr == key
180 +
181 + return False
182 +
183 +
184 +def _is_var_or_elems_access(
185 + node: jinja2.nodes.Node,
186 + varname: str,
187 + key: Optional[str] = None,
188 +) -> bool:
189 + if isinstance(node, jinja2.nodes.Filter):
190 + return (node.node is not None
191 + and _is_var_or_elems_access(node.node, varname, key))
192 + if isinstance(node, jinja2.nodes.Test):
193 + return _is_var_or_elems_access(node.node, varname, key)
194 +
195 + if (isinstance(node, jinja2.nodes.Getitem)
196 + and isinstance(node.arg, jinja2.nodes.Slice)):
197 + return _is_var_or_elems_access(node.node, varname, key)
198 +
199 + # yapf: disable
200 + return (
201 + _is_attr_access(node, varname, key) if key
202 + else _is_var_access(node, varname)
203 + ) # yapf: enable
204 +
205 +
206 +def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
207 + # Global variable that is implicitly defined at the root
208 + yield root, varname
209 +
210 + # Iterative BFS
211 + related_varnames = deque([varname])
212 + while related_varnames:
213 + related_varname = related_varnames.popleft()
214 +
215 + for assign_ast in root.find_all(jinja2.nodes.Assign):
216 + lhs = assign_ast.target
217 + rhs = assign_ast.node
218 +
219 + if _is_var_or_elems_access(rhs, related_varname):
220 + assert isinstance(lhs, jinja2.nodes.Name)
221 + yield assign_ast, lhs.name
222 +
223 + # Avoid infinite looping for self-assignment
224 + if lhs.name != related_varname:
225 + related_varnames.append(lhs.name)
226 +
227 +
228 +# NOTE: The proper way to handle this is to build a CFG so that we can handle
229 +# the scope in which each variable is defined, but that is too complicated
230 +def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
231 + messages_varnames = [
232 + varname
233 + for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
234 + ]
235 +
236 + # Search for {%- for message in messages -%} loops
237 + for loop_ast in root.find_all(jinja2.nodes.For):
238 + loop_iter = loop_ast.iter
239 + loop_target = loop_ast.target
240 +
241 + for varname in messages_varnames:
242 + if _is_var_or_elems_access(loop_iter, varname):
243 + assert isinstance(loop_target, jinja2.nodes.Name)
244 + yield loop_ast, loop_target.name
245 + break
246 +
247 +
248 +def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
249 + message_varnames = [
250 + varname for _, varname in _iter_nodes_assign_messages_item(root)
251 + ]
252 +
253 + # Search for {%- for content in message['content'] -%} loops
254 + for loop_ast in root.find_all(jinja2.nodes.For):
255 + loop_iter = loop_ast.iter
256 + loop_target = loop_ast.target
257 +
258 + for varname in message_varnames:
259 + if _is_var_or_elems_access(loop_iter, varname, "content"):
260 + assert isinstance(loop_target, jinja2.nodes.Name)
261 + yield loop_ast, loop_target.name
262 + break
263 +
264 +
265 +def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
266 + try:
267 + jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
268 + return jinja_compiled.environment.parse(chat_template)
269 + except Exception:
270 + logger.exception("Error when compiling Jinja template")
271 + return None
272 +
273 +
274 +def _detect_content_format(
275 + chat_template: str,
276 + *,
277 + default: _ChatTemplateContentFormat,
278 +) -> _ChatTemplateContentFormat:
279 + jinja_ast = _try_extract_ast(chat_template)
280 + if jinja_ast is None:
281 + return default
282 +
283 + try:
284 + next(_iter_nodes_assign_content_item(jinja_ast))
285 + except StopIteration:
286 + return "string"
287 + except Exception:
288 + logger.exception("Error when parsing AST of Jinja template")
289 + return default
290 + else:
291 + return "openai"
292 +
293 +
294 +def _resolve_chat_template_content_format(
295 + chat_template: Optional[str],
296 + given_format: ChatTemplateContentFormatOption,
297 + tokenizer: AnyTokenizer,
298 +) -> _ChatTemplateContentFormat:
299 + if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
300 + tokenizer_chat_template = tokenizer.chat_template
301 + else:
302 + tokenizer_chat_template = None
303 +
304 + jinja_text: Optional[str]
305 + if isinstance(tokenizer_chat_template, str) and chat_template is None:
306 + jinja_text = tokenizer_chat_template
307 + elif (isinstance(tokenizer_chat_template, dict)
308 + and chat_template in tokenizer_chat_template):
309 + jinja_text = tokenizer_chat_template[chat_template]
310 + else:
311 + jinja_text = load_chat_template(chat_template, is_literal=True)
312 +
313 + detected_format = ("string" if jinja_text is None else
314 + _detect_content_format(jinja_text, default="string"))
315 +
316 + return detected_format if given_format == "auto" else given_format
317 +
318 +
319 +@lru_cache
320 +def resolve_chat_template_content_format(
321 + chat_template: Optional[str],
322 + given_format: ChatTemplateContentFormatOption,
323 + tokenizer: AnyTokenizer,
324 +) -> _ChatTemplateContentFormat:
325 + detected_format = _resolve_chat_template_content_format(
326 + chat_template,
327 + given_format,
328 + tokenizer,
329 + )
330 +
331 + logger.info(
332 + "Detected the chat template content format to be '%s'. "
333 + "You can set `--chat-template-content-format` to override this.",
334 + detected_format,
335 + )
336 +
337 + if given_format != "auto" and given_format != detected_format:
338 + logger.warning(
339 + "You specified `--chat-template-content-format %s` "
340 + "which is different from the detected format '%s'. "
341 + "If our automatic detection is incorrect, please consider "
342 + "opening a GitHub issue so that we can improve it: "
343 + "https://github.com/vllm-project/vllm/issues/new/choose",
344 + given_format,
345 + detected_format,
346 + )
347 +
348 + return detected_format
349 +
350 +
351 ModalityStr = Literal["image", "audio", "video"]
352 _T = TypeVar("_T")
353
@@ -407,12 +602,23 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):
602
603
604 def load_chat_template(
410 - chat_template: Optional[Union[Path, str]]) -> Optional[str]:
605 + chat_template: Optional[Union[Path, str]],
606 + *,
607 + is_literal: bool = False,
608 +) -> Optional[str]:
609 if chat_template is None:
610 return None
611 +
612 + if is_literal:
613 + if isinstance(chat_template, Path):
614 + raise TypeError("chat_template is expected to be read directly "
615 + "from its value")
616 +
617 + return codecs.decode(chat_template, "unicode_escape")
618 +
619 try:
620 with open(chat_template) as f:
415 - resolved_chat_template = f.read()
621 + return f.read()
622 except OSError as e:
623 if isinstance(chat_template, Path):
624 raise
@@ -426,10 +632,7 @@ def load_chat_template(
632
633 # If opening a file fails, set chat template to be args to
634 # ensure we decode so our escape are interpreted correctly
429 - resolved_chat_template = codecs.decode(chat_template, "unicode_escape")
430 -
431 - logger.info("Using supplied chat template:\n%s", resolved_chat_template)
432 - return resolved_chat_template
635 + return load_chat_template(chat_template, is_literal=True)
636
637
638 # TODO: Let user specify how to insert multimodal tokens into prompt
@@ -464,7 +667,6 @@ _ImageParser = partial(cast, ChatCompletionContentPartImageParam)
667 _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
668 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
669 _VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
467 -MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
670
671 # Define a mapping from part types to their corresponding parsing functions.
672 MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
@@ -542,18 +744,12 @@ def _parse_chat_message_content_parts(
744 role: str,
745 parts: Iterable[ChatCompletionContentPartParam],
746 mm_tracker: BaseMultiModalItemTracker,
545 - chat_template_text_format: str,
747 + *,
748 + wrap_dicts: bool,
749 ) -> List[ConversationMessage]:
750 content: List[Union[str, Dict[str, str]]] = []
751
752 mm_parser = mm_tracker.create_parser()
550 - model_config = mm_tracker.model_config
551 -
552 - wrap_dicts = (chat_template_text_format == "openai"
553 - or (model_config.task == "embedding"
554 - and model_config.is_multimodal_model)
555 - or (model_config.hf_config.model_type
556 - in MODEL_KEEP_MULTI_MODAL_CONTENT))
753
754 for part in parts:
755 parse_res = _parse_chat_message_content_part(
@@ -578,9 +774,11 @@ def _parse_chat_message_content_parts(
774
775
776 def _parse_chat_message_content_part(
581 - part: ChatCompletionContentPartParam,
582 - mm_parser: BaseMultiModalContentParser,
583 - wrap_dicts: bool) -> Optional[Union[str, Dict[str, str]]]:
777 + part: ChatCompletionContentPartParam,
778 + mm_parser: BaseMultiModalContentParser,
779 + *,
780 + wrap_dicts: bool,
781 +) -> Optional[Union[str, Dict[str, str]]]:
782 """Parses a single part of a conversation. If wrap_dicts is True,
783 structured dictionary pieces for texts and images will be
784 wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
@@ -629,7 +827,7 @@ _ToolParser = partial(cast, ChatCompletionToolMessageParam)
827 def _parse_chat_message_content(
828 message: ChatCompletionMessageParam,
829 mm_tracker: BaseMultiModalItemTracker,
632 - chat_template_text_format: str,
830 + content_format: _ChatTemplateContentFormat,
831 ) -> List[ConversationMessage]:
832 role = message["role"]
833 content = message.get("content")
@@ -645,7 +843,7 @@ def _parse_chat_message_content(
843 role,
844 content, # type: ignore
845 mm_tracker,
648 - chat_template_text_format,
846 + wrap_dicts=(content_format == "openai"),
847 )
848
849 for result_msg in result:
@@ -684,6 +882,7 @@ def parse_chat_messages(
882 messages: List[ChatCompletionMessageParam],
883 model_config: ModelConfig,
884 tokenizer: AnyTokenizer,
885 + content_format: _ChatTemplateContentFormat,
886 ) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:
887 conversation: List[ConversationMessage] = []
888 mm_tracker = MultiModalItemTracker(model_config, tokenizer)
@@ -692,7 +891,7 @@ def parse_chat_messages(
891 sub_messages = _parse_chat_message_content(
892 msg,
893 mm_tracker,
695 - model_config.chat_template_text_format,
894 + content_format,
895 )
896
897 conversation.extend(sub_messages)
@@ -706,6 +905,7 @@ def parse_chat_messages_futures(
905 messages: List[ChatCompletionMessageParam],
906 model_config: ModelConfig,
907 tokenizer: AnyTokenizer,
908 + content_format: _ChatTemplateContentFormat,
909 ) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
910 conversation: List[ConversationMessage] = []
911 mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
@@ -714,7 +914,7 @@ def parse_chat_messages_futures(
914 sub_messages = _parse_chat_message_content(
915 msg,
916 mm_tracker,
717 - model_config.chat_template_text_format,
917 + content_format,
918 )
919
920 conversation.extend(sub_messages)
vllm/entrypoints/llm.py
+33 -11
@@ -13,9 +13,11 @@ from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
13 TaskOption)
14 from vllm.engine.llm_engine import LLMEngine
15 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
16 + ChatTemplateContentFormatOption,
17 apply_hf_chat_template,
18 apply_mistral_chat_template,
18 - parse_chat_messages)
19 + parse_chat_messages,
20 + resolve_chat_template_content_format)
21 from vllm.inputs import PromptType, TextPrompt, TokensPrompt
22 from vllm.inputs.parse import parse_and_batch_prompt
23 from vllm.logger import init_logger
@@ -523,6 +525,7 @@ class LLM:
525 use_tqdm: bool = True,
526 lora_request: Optional[LoRARequest] = None,
527 chat_template: Optional[str] = None,
528 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",
529 add_generation_prompt: bool = True,
530 continue_final_message: bool = False,
531 tools: Optional[List[Dict[str, Any]]] = None,
@@ -539,9 +542,11 @@ class LLM:
542 to the OpenAI API.
543
544 Args:
542 - messages: A list of conversations or a single conversation.
543 - - Each conversation is represented as a list of messages.
544 - - Each message is a dictionary with 'role' and 'content' keys.
545 + messages: A list of conversations or a single conversation.
546 +
547 + - Each conversation is represented as a list of messages.
548 + - Each message is a dictionary with 'role' and 'content' keys.
549 +
550 sampling_params: The sampling parameters for text generation.
551 If None, we use the default sampling parameters. When it
552 is a single value, it is applied to every prompt. When it
@@ -551,11 +556,19 @@ class LLM:
556 lora_request: LoRA request to use for generation, if any.
557 chat_template: The template to use for structuring the chat.
558 If not provided, the model's default chat template will be used.
559 + chat_template_content_format: The format to render message content.
560 +
561 + - "string" will render the content as a string.
562 + Example: ``"Who are you?"``
563 + - "openai" will render the content as a list of dictionaries,
564 + similar to OpenAI schema.
565 + Example: ``[{"type": "text", "text": "Who are you?"}]``
566 +
567 add_generation_prompt: If True, adds a generation template
568 to each message.
569 continue_final_message: If True, continues the final message in
557 - the conversation instead of starting a new one. Cannot be `True`
558 - if `add_generation_prompt` is also `True`.
570 + the conversation instead of starting a new one. Cannot be
571 + ``True`` if ``add_generation_prompt`` is also ``True``.
572 mm_processor_kwargs: Multimodal processor kwarg overrides for this
573 chat request. Only used for offline requests.
574
@@ -576,17 +589,26 @@ class LLM:
589 cast(List[ChatCompletionMessageParam], messages)
590 ]
591
592 + tokenizer = self.get_tokenizer()
593 + model_config = self.llm_engine.get_model_config()
594 + resolved_content_format = resolve_chat_template_content_format(
595 + chat_template,
596 + chat_template_content_format,
597 + tokenizer,
598 + )
599 +
600 prompts: List[Union[TokensPrompt, TextPrompt]] = []
601
602 for msgs in list_of_messages:
582 - tokenizer = self.get_tokenizer()
583 - model_config = self.llm_engine.get_model_config()
584 -
603 # NOTE: _parse_chat_message_content_parts() currently doesn't
604 # handle mm_processor_kwargs, since there is no implementation in
605 # the chat message parsing for it.
606 conversation, mm_data = parse_chat_messages(
589 - msgs, model_config, tokenizer)
607 + msgs,
608 + model_config,
609 + tokenizer,
610 + content_format=resolved_content_format,
611 + )
612
613 prompt_data: Union[str, List[int]]
614 if isinstance(tokenizer, MistralTokenizer):
@@ -737,7 +759,7 @@ class LLM:
759 generation, if any.
760
761 Returns:
740 - A list of `EmbeddingRequestOutput` objects containing the
762 + A list of ``EmbeddingRequestOutput`` objects containing the
763 generated embeddings in the same order as the input prompts.
764
765 Note:
vllm/entrypoints/openai/api_server.py
+10 -3
@@ -29,6 +29,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
29 from vllm.engine.multiprocessing.client import MQLLMEngineClient
30 from vllm.engine.multiprocessing.engine import run_mp_engine
31 from vllm.engine.protocol import EngineClient
32 +from vllm.entrypoints.chat_utils import load_chat_template
33 from vllm.entrypoints.launcher import serve_http
34 from vllm.entrypoints.logger import RequestLogger
35 from vllm.entrypoints.openai.cli_args import (make_arg_parser,
@@ -529,6 +530,9 @@ def init_app_state(
530 state.engine_client = engine_client
531 state.log_stats = not args.disable_log_stats
532
533 + resolved_chat_template = load_chat_template(args.chat_template)
534 + logger.info("Using supplied chat template:\n%s", resolved_chat_template)
535 +
536 state.openai_serving_chat = OpenAIServingChat(
537 engine_client,
538 model_config,
@@ -537,7 +541,8 @@ def init_app_state(
541 lora_modules=args.lora_modules,
542 prompt_adapters=args.prompt_adapters,
543 request_logger=request_logger,
540 - chat_template=args.chat_template,
544 + chat_template=resolved_chat_template,
545 + chat_template_content_format=args.chat_template_content_format,
546 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
547 enable_auto_tools=args.enable_auto_tool_choice,
548 tool_parser=args.tool_call_parser,
@@ -557,7 +562,8 @@ def init_app_state(
562 model_config,
563 base_model_paths,
564 request_logger=request_logger,
560 - chat_template=args.chat_template,
565 + chat_template=resolved_chat_template,
566 + chat_template_content_format=args.chat_template_content_format,
567 ) if model_config.task == "embedding" else None
568 state.openai_serving_tokenization = OpenAIServingTokenization(
569 engine_client,
@@ -565,7 +571,8 @@ def init_app_state(
571 base_model_paths,
572 lora_modules=args.lora_modules,
573 request_logger=request_logger,
568 - chat_template=args.chat_template,
574 + chat_template=resolved_chat_template,
575 + chat_template_content_format=args.chat_template_content_format,
576 )
577
578
vllm/entrypoints/openai/cli_args.py
+15 -2
@@ -7,10 +7,11 @@ purposes.
7 import argparse
8 import json
9 import ssl
10 -from typing import List, Optional, Sequence, Union
10 +from typing import List, Optional, Sequence, Union, get_args
11
12 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13 -from vllm.entrypoints.chat_utils import validate_chat_template
13 +from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14 + validate_chat_template)
15 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
16 PromptAdapterPath)
17 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -132,6 +133,18 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
133 help="The file path to the chat template, "
134 "or the template in single-line form "
135 "for the specified model")
136 + parser.add_argument(
137 + '--chat-template-content-format',
138 + type=str,
139 + default="auto",
140 + choices=get_args(ChatTemplateContentFormatOption),
141 + help='The format to render message content within a chat template.'
142 + '\n\n'
143 + '* "string" will render the content as a string. '
144 + 'Example: "Hello World"\n'
145 + '* "openai" will render the content as a list of dictionaries, '
146 + 'similar to OpenAI schema. '
147 + 'Example: [{"type": "text", "text": "Hello world!"}]')
148 parser.add_argument("--response-role",
149 type=nullable_str,
150 default="assistant",
vllm/entrypoints/openai/protocol.py
+45 -26
@@ -5,9 +5,8 @@ from argparse import Namespace
5 from typing import Any, Dict, List, Literal, Optional, Union
6
7 import torch
8 -from openai.types.chat import ChatCompletionContentPartParam
8 from pydantic import BaseModel, ConfigDict, Field, model_validator
10 -from typing_extensions import Annotated, Required, TypedDict
9 +from typing_extensions import Annotated
10
11 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12 from vllm.pooling_params import PoolingParams
@@ -35,26 +34,6 @@ assert _LONG_INFO.min == _MOCK_LONG_INFO.min
34 assert _LONG_INFO.max == _MOCK_LONG_INFO.max
35
36
38 -class CustomChatCompletionMessageParam(TypedDict, total=False):
39 - """Enables custom roles in the Chat Completion API."""
40 - role: Required[str]
41 - """The role of the message's author."""
42 -
43 - content: Union[str, List[ChatCompletionContentPartParam]]
44 - """The contents of the message."""
45 -
46 - name: str
47 - """An optional name for the participant.
48 -
49 - Provides the model information to differentiate between participants of the
50 - same role.
51 - """
52 -
53 - tool_call_id: Optional[str]
54 -
55 - tool_calls: Optional[List[dict]]
56 -
57 -
37 class OpenAIBaseModel(BaseModel):
38 # OpenAI API does not allow extra fields
39 model_config = ConfigDict(extra="forbid")
@@ -1054,16 +1033,56 @@ class TokenizeCompletionRequest(OpenAIBaseModel):
1033 model: str
1034 prompt: str
1035
1057 - add_special_tokens: bool = Field(default=True)
1036 + add_special_tokens: bool = Field(
1037 + default=True,
1038 + description=(
1039 + "If true (the default), special tokens (e.g. BOS) will be added to "
1040 + "the prompt."),
1041 + )
1042
1043
1044 class TokenizeChatRequest(OpenAIBaseModel):
1045 model: str
1046 messages: List[ChatCompletionMessageParam]
1047
1064 - add_generation_prompt: bool = Field(default=True)
1065 - continue_final_message: bool = Field(default=False)
1066 - add_special_tokens: bool = Field(default=False)
1048 + add_generation_prompt: bool = Field(
1049 + default=True,
1050 + description=
1051 + ("If true, the generation prompt will be added to the chat template. "
1052 + "This is a parameter used by chat template in tokenizer config of the "
1053 + "model."),
1054 + )
1055 + continue_final_message: bool = Field(
1056 + default=False,
1057 + description=
1058 + ("If this is set, the chat will be formatted so that the final "
1059 + "message in the chat is open-ended, without any EOS tokens. The "
1060 + "model will continue this message rather than starting a new one. "
1061 + "This allows you to \"prefill\" part of the model's response for it. "
1062 + "Cannot be used at the same time as `add_generation_prompt`."),
1063 + )
1064 + add_special_tokens: bool = Field(
1065 + default=False,
1066 + description=(
1067 + "If true, special tokens (e.g. BOS) will be added to the prompt "
1068 + "on top of what is added by the chat template. "
1069 + "For most models, the chat template takes care of adding the "
1070 + "special tokens so this should be set to false (as is the "
1071 + "default)."),
1072 + )
1073 + chat_template: Optional[str] = Field(
1074 + default=None,
1075 + description=(
1076 + "A Jinja template to use for this conversion. "
1077 + "As of transformers v4.44, default chat template is no longer "
1078 + "allowed, so you must provide a chat template if the tokenizer "
1079 + "does not define one."),
1080 + )
1081 + chat_template_kwargs: Optional[Dict[str, Any]] = Field(
1082 + default=None,
1083 + description=("Additional kwargs to pass to the template renderer. "
1084 + "Will be accessible by the chat template."),
1085 + )
1086
1087 @model_validator(mode="before")
1088 @classmethod
vllm/entrypoints/openai/run_batch.py
+2
@@ -222,6 +222,7 @@ async def main(args):
222 prompt_adapters=None,
223 request_logger=request_logger,
224 chat_template=None,
225 + chat_template_content_format="auto",
226 enable_prompt_tokens_details=args.enable_prompt_tokens_details,
227 ) if model_config.task == "generate" else None
228 openai_serving_embedding = OpenAIServingEmbedding(
@@ -230,6 +231,7 @@ async def main(args):
231 base_model_paths,
232 request_logger=request_logger,
233 chat_template=None,
234 + chat_template_content_format="auto",
235 ) if model_config.task == "embedding" else None
236
237 tracker = BatchProgressTracker()
vllm/entrypoints/openai/serving_chat.py
+23 -17
@@ -10,7 +10,8 @@ from fastapi import Request
10
11 from vllm.config import ModelConfig
12 from vllm.engine.protocol import EngineClient
13 -from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
13 +from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14 + ConversationMessage)
15 from vllm.entrypoints.logger import RequestLogger
16 from vllm.entrypoints.openai.protocol import (
17 ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -38,20 +39,23 @@ logger = init_logger(__name__)
39
40 class OpenAIServingChat(OpenAIServing):
41
41 - def __init__(self,
42 - engine_client: EngineClient,
43 - model_config: ModelConfig,
44 - base_model_paths: List[BaseModelPath],
45 - response_role: str,
46 - *,
47 - lora_modules: Optional[List[LoRAModulePath]],
48 - prompt_adapters: Optional[List[PromptAdapterPath]],
49 - request_logger: Optional[RequestLogger],
50 - chat_template: Optional[str],
51 - return_tokens_as_token_ids: bool = False,
52 - enable_auto_tools: bool = False,
53 - tool_parser: Optional[str] = None,
54 - enable_prompt_tokens_details: bool = False):
42 + def __init__(
43 + self,
44 + engine_client: EngineClient,
45 + model_config: ModelConfig,
46 + base_model_paths: List[BaseModelPath],
47 + response_role: str,
48 + *,
49 + lora_modules: Optional[List[LoRAModulePath]],
50 + prompt_adapters: Optional[List[PromptAdapterPath]],
51 + request_logger: Optional[RequestLogger],
52 + chat_template: Optional[str],
53 + chat_template_content_format: ChatTemplateContentFormatOption,
54 + return_tokens_as_token_ids: bool = False,
55 + enable_auto_tools: bool = False,
56 + tool_parser: Optional[str] = None,
57 + enable_prompt_tokens_details: bool = False,
58 + ) -> None:
59 super().__init__(engine_client=engine_client,
60 model_config=model_config,
61 base_model_paths=base_model_paths,
@@ -61,8 +65,8 @@ class OpenAIServingChat(OpenAIServing):
65 return_tokens_as_token_ids=return_tokens_as_token_ids)
66
67 self.response_role = response_role
64 - self.use_tool_use_model_template = False
65 - self.chat_template = load_chat_template(chat_template)
68 + self.chat_template = chat_template
69 + self.chat_template_content_format: Final = chat_template_content_format
70
71 # set up tool use
72 self.enable_auto_tools: bool = enable_auto_tools
@@ -120,6 +124,7 @@ class OpenAIServingChat(OpenAIServing):
124 ) = self._maybe_get_adapters(request)
125
126 tokenizer = await self.engine_client.get_tokenizer(lora_request)
127 +
128 tool_parser = self.tool_parser
129
130 # validation for OpenAI tools
@@ -157,6 +162,7 @@ class OpenAIServingChat(OpenAIServing):
162 tokenizer,
163 request.messages,
164 chat_template=request.chat_template or self.chat_template,
165 + chat_template_content_format=self.chat_template_content_format,
166 add_generation_prompt=request.add_generation_prompt,
167 continue_final_message=request.continue_final_message,
168 tool_dicts=tool_dicts,
vllm/entrypoints/openai/serving_embedding.py
+8 -4
@@ -1,7 +1,7 @@
1 import asyncio
2 import base64
3 import time
4 -from typing import AsyncGenerator, List, Literal, Optional, Union, cast
4 +from typing import AsyncGenerator, Final, List, Literal, Optional, Union, cast
5
6 import numpy as np
7 from fastapi import Request
@@ -9,7 +9,7 @@ from typing_extensions import assert_never
9
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import EngineClient
12 -from vllm.entrypoints.chat_utils import load_chat_template
12 +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13 from vllm.entrypoints.logger import RequestLogger
14 from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
15 EmbeddingRequest,
@@ -77,7 +77,8 @@ class OpenAIServingEmbedding(OpenAIServing):
77 *,
78 request_logger: Optional[RequestLogger],
79 chat_template: Optional[str],
80 - ):
80 + chat_template_content_format: ChatTemplateContentFormatOption,
81 + ) -> None:
82 super().__init__(engine_client=engine_client,
83 model_config=model_config,
84 base_model_paths=base_model_paths,
@@ -85,7 +86,8 @@ class OpenAIServingEmbedding(OpenAIServing):
86 prompt_adapters=None,
87 request_logger=request_logger)
88
88 - self.chat_template = load_chat_template(chat_template)
89 + self.chat_template = chat_template
90 + self.chat_template_content_format: Final = chat_template_content_format
91
92 async def create_embedding(
93 self,
@@ -144,6 +146,8 @@ class OpenAIServingEmbedding(OpenAIServing):
146 tokenizer,
147 request.messages,
148 chat_template=request.chat_template or self.chat_template,
149 + chat_template_content_format=self.
150 + chat_template_content_format,
151 add_generation_prompt=request.add_generation_prompt,
152 continue_final_message=request.continue_final_message,
153 truncate_prompt_tokens=truncate_prompt_tokens,
vllm/entrypoints/openai/serving_engine.py
+13 -4
@@ -11,14 +11,16 @@ from typing_extensions import Annotated
11
12 from vllm.config import ModelConfig
13 from vllm.engine.protocol import EngineClient
14 +# yapf conflicts with isort for this block
15 +# yapf: disable
16 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
17 + ChatTemplateContentFormatOption,
18 ConversationMessage,
19 apply_hf_chat_template,
20 apply_mistral_chat_template,
18 - parse_chat_messages_futures)
21 + parse_chat_messages_futures,
22 + resolve_chat_template_content_format)
23 from vllm.entrypoints.logger import RequestLogger
20 -# yapf conflicts with isort for this block
21 -# yapf: disable
24 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
25 CompletionRequest,
26 DetokenizeRequest,
@@ -426,7 +428,8 @@ class OpenAIServing:
428 request: ChatLikeRequest,
429 tokenizer: AnyTokenizer,
430 messages: List[ChatCompletionMessageParam],
429 - chat_template: Optional[str] = None,
431 + chat_template: Optional[str],
432 + chat_template_content_format: ChatTemplateContentFormatOption,
433 add_generation_prompt: bool = True,
434 continue_final_message: bool = False,
435 tool_dicts: Optional[List[Dict[str, Any]]] = None,
@@ -437,10 +440,16 @@ class OpenAIServing:
440 add_special_tokens: bool = False,
441 ) -> Tuple[List[ConversationMessage], Sequence[RequestPrompt],
442 List[TokensPrompt]]:
443 + resolved_content_format = resolve_chat_template_content_format(
444 + chat_template,
445 + chat_template_content_format,
446 + tokenizer,
447 + )
448 conversation, mm_data_future = parse_chat_messages_futures(
449 messages,
450 self.model_config,
451 tokenizer,
452 + content_format=resolved_content_format,
453 )
454
455 _chat_template_kwargs: Dict[str, Any] = dict(
vllm/entrypoints/openai/serving_tokenization.py
+10 -10
@@ -1,8 +1,8 @@
1 -from typing import List, Optional, Union
1 +from typing import Final, List, Optional, Union
2
3 from vllm.config import ModelConfig
4 from vllm.engine.protocol import EngineClient
5 -from vllm.entrypoints.chat_utils import load_chat_template
5 +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
6 from vllm.entrypoints.logger import RequestLogger
7 # yapf conflicts with isort for this block
8 # yapf: disable
@@ -33,7 +33,8 @@ class OpenAIServingTokenization(OpenAIServing):
33 lora_modules: Optional[List[LoRAModulePath]],
34 request_logger: Optional[RequestLogger],
35 chat_template: Optional[str],
36 - ):
36 + chat_template_content_format: ChatTemplateContentFormatOption,
37 + ) -> None:
38 super().__init__(engine_client=engine_client,
39 model_config=model_config,
40 base_model_paths=base_model_paths,
@@ -41,12 +42,8 @@ class OpenAIServingTokenization(OpenAIServing):
42 prompt_adapters=None,
43 request_logger=request_logger)
44
44 - # If this is None we use the tokenizer's default chat template
45 - # the list of commonly-used chat template names for HF named templates
46 - hf_chat_templates: List[str] = ['default', 'tool_use']
47 - self.chat_template = chat_template \
48 - if chat_template in hf_chat_templates \
49 - else load_chat_template(chat_template)
45 + self.chat_template = chat_template
46 + self.chat_template_content_format: Final = chat_template_content_format
47
48 async def create_tokenize(
49 self,
@@ -75,9 +72,12 @@ class OpenAIServingTokenization(OpenAIServing):
72 request,
73 tokenizer,
74 request.messages,
78 - chat_template=self.chat_template,
75 + chat_template=request.chat_template or self.chat_template,
76 + chat_template_content_format=self.
77 + chat_template_content_format,
78 add_generation_prompt=request.add_generation_prompt,
79 continue_final_message=request.continue_final_message,
80 + chat_template_kwargs=request.chat_template_kwargs,
81 add_special_tokens=request.add_special_tokens,
82 )
83 else: