@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f399182e8c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Run ruff format on a few files. (#24075)

Signed-off-by: Chenheli Hua <huachenheli@outlook.com>

Chenheli Hua committed Sep 2, 2025 at 10:55 UTC f399182e8c401303f7399b953417b926e4eb64f2

3 files changed +947 -690

tests/entrypoints/test_chat_utils.py

+578 -452

@@ -46,23 +46,27 @@ MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

47 @pytest.fixture(scope="function")

48 def phi3v_model_config():

49 - return ModelConfig(PHI3V_MODEL_ID,

50 - runner="generate",

51 - trust_remote_code=True,

52 - limit_mm_per_prompt={

53 - "image": 2,

54 - })

49 + return ModelConfig(

50 + PHI3V_MODEL_ID,

51 + runner="generate",

52 + trust_remote_code=True,

53 + limit_mm_per_prompt={

54 + "image": 2,

55 + },

56 + )

59 @pytest.fixture(scope="function")

60 def phi3v_model_config_mm_interleaved():

59 - return ModelConfig(PHI3V_MODEL_ID,

60 - runner="generate",

61 - trust_remote_code=True,

62 - interleave_mm_strings=True,

63 - limit_mm_per_prompt={

64 - "image": 2,

65 - })

61 + return ModelConfig(

62 + PHI3V_MODEL_ID,

63 + runner="generate",

64 + trust_remote_code=True,

65 + interleave_mm_strings=True,

66 + limit_mm_per_prompt={

67 + "image": 2,

68 + },

69 + )

72 @pytest.fixture(scope="module")

@@ -77,14 +81,16 @@ def phi3v_tokenizer():

82 @pytest.fixture(scope="function")

83 def qwen25omni_model_config_mm_interleaved():

80 - return ModelConfig(QWEN25OMNI_MODEL_ID,

81 - runner="generate",

82 - interleave_mm_strings=True,

83 - limit_mm_per_prompt={

84 - "image": 2,

85 - "audio": 1,

86 - "video": 1,

87 - })

84 + return ModelConfig(

85 + QWEN25OMNI_MODEL_ID,

86 + runner="generate",

87 + interleave_mm_strings=True,

88 + limit_mm_per_prompt={

89 + "image": 2,

90 + "audio": 1,

91 + "video": 1,

92 + },

93 + )

96 @pytest.fixture(scope="module")

@@ -99,11 +105,13 @@ def qwen25omni_tokenizer():

105

106 @pytest.fixture(scope="module")

107 def mllama_model_config():

102 - return ModelConfig(MLLAMA_MODEL_ID,

103 - runner="generate",

104 - limit_mm_per_prompt={

105 - "image": 2,

106 - })

108 + return ModelConfig(

109 + MLLAMA_MODEL_ID,

110 + runner="generate",

111 + limit_mm_per_prompt={

112 + "image": 2,

113 + },

114 + )

115

116

117 @pytest.fixture(scope="module")

@@ -118,11 +126,13 @@ def mllama_tokenizer():

126

127 @pytest.fixture(scope="function")

128 def mistral_model_config():

121 - return ModelConfig(MISTRAL_MODEL_ID,

122 - runner="generate",

123 - limit_mm_per_prompt={

124 - "image": 2,

125 - })

129 + return ModelConfig(

130 + MISTRAL_MODEL_ID,

131 + runner="generate",

132 + limit_mm_per_prompt={

133 + "image": 2,

134 + },

135 + )

136

137

138 @pytest.fixture(scope="module")

@@ -137,21 +147,21 @@ def mistral_tokenizer():

147

148 @pytest.fixture(scope="module")

149 def image_url():

140 - image = ImageAsset('cherry_blossom')

150 + image = ImageAsset("cherry_blossom")

151 base64 = encode_image_base64(image.pil_image)

152 return f"data:image/jpeg;base64,{base64}"

153

154

155 @pytest.fixture(scope="module")

156 def video_url():

147 - video = VideoAsset('baby_reading', 1)

157 + video = VideoAsset("baby_reading", 1)

158 base64 = encode_video_base64(video.np_ndarrays)

159 return f"data:video/jpeg;base64,{base64}"

160

161

162 @pytest.fixture(scope="module")

163 def audio_url():

154 - audio = AudioAsset('mary_had_lamb')

164 + audio = AudioAsset("mary_had_lamb")

165 base64 = encode_audio_base64(*audio.audio_and_sample_rate)

166 return f"data:audio/ogg;base64,{base64}"

167

@@ -195,15 +205,18 @@ def test_parse_chat_messages_single_image(

205 [{

206 "role":

207 "user",

198 - "content": [{

199 - "type": "image_url",

200 - "image_url": {

201 - "url": image_url

202 - }

203 - }, {

204 - "type": "text",

205 - "text": "What's in the image?"

206 - }]

208 + "content": [

209 + {

210 + "type": "image_url",

211 + "image_url": {

212 + "url": image_url

213 + }

214 + },

215 + {

216 + "type": "text",

217 + "text": "What's in the image?"

218 + },

219 + ],

220 }],

221 phi3v_model_config,

222 phi3v_tokenizer,

@@ -223,58 +236,69 @@ def test_parse_chat_messages_empty_system(

236 ):

237 # Test string format

238 conversation, _ = parse_chat_messages(

226 - [{

227 - "role": "system",

228 - "content": ""

229 - }, {

230 - "role": "user",

231 - "content": [{

232 - "type": "text",

233 - "text": "Who are you?"

234 - }]

235 - }],

239 + [

240 + {

241 + "role": "system",

242 + "content": ""

243 + },

244 + {

245 + "role": "user",

246 + "content": [{

247 + "type": "text",

248 + "text": "Who are you?"

249 + }],

250 + },

251 + ],

252 mistral_model_config,

253 mistral_tokenizer,

254 content_format="string",

255 )

240 - assert conversation == [{

241 - "role": "system",

242 - "content": ""

243 - }, {

244 - "role": "user",

245 - "content": "Who are you?"

246 - }]

256 + assert conversation == [

257 + {

258 + "role": "system",

259 + "content": ""

260 + },

261 + {

262 + "role": "user",

263 + "content": "Who are you?"

264 + },

265 + ]

266

267 # Test openai format

268 conversation, _ = parse_chat_messages(

250 - [{

269 + [

270 + {

271 + "role": "system",

272 + "content": ""

273 + },

274 + {

275 + "role": "user",

276 + "content": [{

277 + "type": "text",

278 + "text": "Who are you?"

279 + }],

280 + },

281 + ],

282 + mistral_model_config,

283 + mistral_tokenizer,

284 + content_format="openai",

285 + )

286 + assert conversation == [

287 + {

288 "role": "system",

252 - "content": ""

253 - }, {

289 + "content": [{

290 + "type": "text",

291 + "text": ""

292 + }]

293 + },

294 + {

295 "role": "user",

296 "content": [{

297 "type": "text",

298 "text": "Who are you?"

299 }]

259 - }],

260 - mistral_model_config,

261 - mistral_tokenizer,

262 - content_format="openai",

263 - )

264 - assert conversation == [{

265 - "role": "system",

266 - "content": [{

267 - "type": "text",

268 - "text": ""

269 - }]

270 - }, {

271 - "role":

272 - "user",

273 - "content": [{

274 - "type": "text",

275 - "text": "Who are you?"

276 - }]

277 - }]

300 + },

301 + ]

302

303

304 @pytest.mark.asyncio

@@ -287,15 +311,18 @@ async def test_parse_chat_messages_single_image_async(

311 [{

312 "role":

313 "user",

290 - "content": [{

291 - "type": "image_url",

292 - "image_url": {

293 - "url": image_url

294 - }

295 - }, {

296 - "type": "text",

297 - "text": "What's in the image?"

298 - }]

314 + "content": [

315 + {

316 + "type": "image_url",

317 + "image_url": {

318 + "url": image_url

319 + }

320 + },

321 + {

322 + "type": "text",

323 + "text": "What's in the image?"

324 + },

325 + ],

326 }],

327 phi3v_model_config,

328 phi3v_tokenizer,

@@ -318,18 +345,22 @@ def test_parse_chat_messages_multiple_images(

345 [{

346 "role":

347 "user",

321 - "content": [{

322 - "type": "image_url",

323 - "image_url": {

324 - "url": image_url

325 - }

326 - }, {

327 - "type": "image_pil",

328 - "image_pil": ImageAsset('cherry_blossom').pil_image

329 - }, {

330 - "type": "text",

331 - "text": "What's in these images?"

332 - }]

348 + "content": [

349 + {

350 + "type": "image_url",

351 + "image_url": {

352 + "url": image_url

353 + }

354 + },

355 + {

356 + "type": "image_pil",

357 + "image_pil": ImageAsset("cherry_blossom").pil_image,

358 + },

359 + {

360 + "type": "text",

361 + "text": "What's in these images?"

362 + },

363 + ],

364 }],

365 phi3v_model_config,

366 phi3v_tokenizer,

@@ -340,7 +371,7 @@ def test_parse_chat_messages_multiple_images(

371 "role":

372 "user",

373 "content":

343 - "<|image_1|>\n<|image_2|>\nWhat's in these images?"

374 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",

375 }]

376 _assert_mm_data_is_image_input(mm_data, 2)

377

@@ -355,18 +386,22 @@ async def test_parse_chat_messages_multiple_images_async(

386 [{

387 "role":

388 "user",

358 - "content": [{

359 - "type": "image_url",

360 - "image_url": {

361 - "url": image_url

362 - }

363 - }, {

364 - "type": "image_pil",

365 - "image_pil": ImageAsset('cherry_blossom').pil_image

366 - }, {

367 - "type": "text",

368 - "text": "What's in these images?"

369 - }]

389 + "content": [

390 + {

391 + "type": "image_url",

392 + "image_url": {

393 + "url": image_url

394 + }

395 + },

396 + {

397 + "type": "image_pil",

398 + "image_pil": ImageAsset("cherry_blossom").pil_image,

399 + },

400 + {

401 + "type": "text",

402 + "text": "What's in these images?"

403 + },

404 + ],

405 }],

406 phi3v_model_config,

407 phi3v_tokenizer,

@@ -377,7 +412,7 @@ async def test_parse_chat_messages_multiple_images_async(

412 "role":

413 "user",

414 "content":

380 - "<|image_1|>\n<|image_2|>\nWhat's in these images?"

415 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",

416 }]

417 _assert_mm_data_is_image_input(await mm_future, 2)

418

@@ -391,22 +426,26 @@ def test_parse_chat_messages_placeholder_already_in_prompt(

426 [{

427 "role":

428 "user",

394 - "content": [{

395 - "type": "image_url",

396 - "image_url": {

397 - "url": image_url

398 - }

399 - }, {

400 - "type": "image_url",

401 - "image_url": {

402 - "url": image_url

403 - }

404 - }, {

405 - "type":

406 - "text",

407 - "text":

408 - "What's in <|image_1|> and how does it compare to <|image_2|>?"

409 - }]

429 + "content": [

430 + {

431 + "type": "image_url",

432 + "image_url": {

433 + "url": image_url

434 + }

435 + },

436 + {

437 + "type": "image_url",

438 + "image_url": {

439 + "url": image_url

440 + }

441 + },

442 + {

443 + "type":

444 + "text",

445 + "text":

446 + "What's in <|image_1|> and how does it compare to <|image_2|>?", # noqa: E501

447 + },

448 + ],

449 }],

450 phi3v_model_config,

451 phi3v_tokenizer,

@@ -416,7 +455,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(

455 "role":

456 "user",

457 "content":

419 - "What's in <|image_1|> and how does it compare to <|image_2|>?"

458 + "What's in <|image_1|> and how does it compare to <|image_2|>?",

459 }]

460 _assert_mm_data_is_image_input(mm_data, 2)

461

@@ -447,9 +486,9 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(

486 "type":

487 "text",

488 "text":

450 - "What's in <|image_1|> and how does it compare to the other one?" # noqa: E501

451 - }

452 - ]

489 + "What's in <|image_1|> and how does it compare to the other one?", # noqa: E501

490 + },

491 + ],

492 }],

493 phi3v_model_config,

494 phi3v_tokenizer,

@@ -461,7 +500,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(

500 "user",

501 "content":

502 "<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "

464 - "other one?"

503 + "other one?",

504 }]

505 _assert_mm_data_is_image_input(mm_data, 2)

506

@@ -472,34 +511,44 @@ def test_parse_chat_messages_multiple_images_across_messages(

511 image_url,

512 ):

513 conversation, mm_data = parse_chat_messages(

475 - [{

476 - "role":

477 - "user",

478 - "content": [{

479 - "type": "image_url",

480 - "image_url": {

481 - "url": image_url

482 - }

483 - }, {

484 - "type": "text",

485 - "text": "What's in this image?"

486 - }]

487 - }, {

488 - "role": "assistant",

489 - "content": "Some stuff."

490 - }, {

491 - "role":

492 - "user",

493 - "content": [{

494 - "type": "image_url",

495 - "image_url": {

496 - "url": image_url

497 - }

498 - }, {

499 - "type": "text",

500 - "text": "What about this one?"

501 - }]

502 - }],

514 + [

515 + {

516 + "role":

517 + "user",

518 + "content": [

519 + {

520 + "type": "image_url",

521 + "image_url": {

522 + "url": image_url

523 + }

524 + },

525 + {

526 + "type": "text",

527 + "text": "What's in this image?"

528 + },

529 + ],

530 + },

531 + {

532 + "role": "assistant",

533 + "content": "Some stuff."

534 + },

535 + {

536 + "role":

537 + "user",

538 + "content": [

539 + {

540 + "type": "image_url",

541 + "image_url": {

542 + "url": image_url

543 + }

544 + },

545 + {

546 + "type": "text",

547 + "text": "What about this one?"

548 + },

549 + ],

550 + },

551 + ],

552 phi3v_model_config,

553 phi3v_tokenizer,

554 content_format="string",

@@ -527,19 +576,23 @@ def test_parse_chat_messages_context_text_format(

576 phi3v_tokenizer,

577 ):

578 conversation, mm_data = parse_chat_messages(

530 - [{

531 - "role": "user",

532 - "content": [{

533 - "type": "text",

534 - "text": "What's in this text?"

535 - }]

536 - }, {

537 - "role": "assistant",

538 - "content": "Some stuff."

539 - }, {

540 - "role": "user",

541 - "content": "What about this one?"

542 - }],

579 + [

580 + {

581 + "role": "user",

582 + "content": [{

583 + "type": "text",

584 + "text": "What's in this text?"

585 + }],

586 + },

587 + {

588 + "role": "assistant",

589 + "content": "Some stuff."

590 + },

591 + {

592 + "role": "user",

593 + "content": "What about this one?"

594 + },

595 + ],

596 phi3v_model_config,

597 phi3v_tokenizer,

598 content_format="openai",

@@ -551,21 +604,21 @@ def test_parse_chat_messages_context_text_format(

604 "content": [{

605 "type": "text",

606 "text": "What's in this text?"

554 - }]

607 + }],

608 },

609 {

610 "role": "assistant",

611 "content": [{

612 "type": "text",

613 "text": "Some stuff."

561 - }]

614 + }],

615 },

616 {

617 "role": "user",

618 "content": [{

619 "type": "text",

620 "text": "What about this one?"

568 - }]

621 + }],

622 },

623 ]

624

@@ -578,31 +631,37 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(

631 with warnings.catch_warnings():

632 warnings.filterwarnings(

633 "ignore",

581 - message="coroutine 'async_get_and_parse_image' was never awaited")

634 + message="coroutine 'async_get_and_parse_image' was never awaited",

635 + )

636 with pytest.raises(ValueError, match="At most"):

637 parse_chat_messages(

638 [{

639 "role":

640 "user",

587 - "content": [{

588 - "type": "image_url",

589 - "image_url": {

590 - "url": image_url

591 - }

592 - }, {

593 - "type": "image_url",

594 - "image_url": {

595 - "url": image_url

596 - }

597 - }, {

598 - "type": "image_url",

599 - "image_url": {

600 - "url": image_url

601 - }

602 - }, {

603 - "type": "text",

604 - "text": "What's in these images?"

605 - }]

641 + "content": [

642 + {

643 + "type": "image_url",

644 + "image_url": {

645 + "url": image_url

646 + },

647 + },

648 + {

649 + "type": "image_url",

650 + "image_url": {

651 + "url": image_url

652 + },

653 + },

654 + {

655 + "type": "image_url",

656 + "image_url": {

657 + "url": image_url

658 + },

659 + },

660 + {

661 + "type": "text",

662 + "text": "What's in these images?"

663 + },

664 + ],

665 }],

666 phi3v_model_config,

667 phi3v_tokenizer,

@@ -618,42 +677,54 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(

677 with warnings.catch_warnings():

678 warnings.filterwarnings(

679 "ignore",

621 - message="coroutine 'async_get_and_parse_image' was never awaited")

680 + message="coroutine 'async_get_and_parse_image' was never awaited",

681 + )

682 with pytest.raises(ValueError, match="At most"):

683 parse_chat_messages(

624 - [{

625 - "role":

626 - "user",

627 - "content": [{

628 - "type": "image_url",

629 - "image_url": {

630 - "url": image_url

631 - }

632 - }, {

633 - "type": "text",

634 - "text": "What's in this image?"

635 - }]

636 - }, {

637 - "role": "assistant",

638 - "content": "Some stuff."

639 - }, {

640 - "role":

641 - "user",

642 - "content": [{

643 - "type": "image_url",

644 - "image_url": {

645 - "url": image_url

646 - }

647 - }, {

648 - "type": "image_url",

649 - "image_url": {

650 - "url": image_url

651 - }

652 - }, {

653 - "type": "text",

654 - "text": "What about these two?"

655 - }]

656 - }],

684 + [

685 + {

686 + "role":

687 + "user",

688 + "content": [

689 + {

690 + "type": "image_url",

691 + "image_url": {

692 + "url": image_url

693 + },

694 + },

695 + {

696 + "type": "text",

697 + "text": "What's in this image?"

698 + },

699 + ],

700 + },

701 + {

702 + "role": "assistant",

703 + "content": "Some stuff."

704 + },

705 + {

706 + "role":

707 + "user",

708 + "content": [

709 + {

710 + "type": "image_url",

711 + "image_url": {

712 + "url": image_url

713 + },

714 + },

715 + {

716 + "type": "image_url",

717 + "image_url": {

718 + "url": image_url

719 + },

720 + },

721 + {

722 + "type": "text",

723 + "text": "What about these two?"

724 + },

725 + ],

726 + },

727 + ],

728 phi3v_model_config,

729 phi3v_tokenizer,

730 content_format="string",

@@ -670,12 +741,14 @@ def test_parse_chat_messages_multiple_images_uncommon_input(

741 "role":

742 "user",

743 "content": [

673 - "What's in these images?", {

744 + "What's in these images?",

745 + {

746 "image_url": image_url

675 - }, {

747 + },

748 + {

749 "image_url": image_url

677 - }

678 - ]

750 + },

751 + ],

752 }],

753 phi3v_model_config,

754 phi3v_tokenizer,

@@ -686,7 +759,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(

759 "role":

760 "user",

761 "content":

689 - "<|image_1|>\n<|image_2|>\nWhat's in these images?"

762 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",

763 }]

764 _assert_mm_data_is_image_input(mm_data, 2)

765

@@ -700,26 +773,32 @@ def test_parse_chat_messages_multiple_images_interleave(

773 [{

774 "role":

775 "user",

703 - "content": [{

704 - "type": "text",

705 - "text": "I need you to compare this image"

706 - }, {

707 - "type": "image_url",

708 - "image_url": {

709 - "url": image_url

710 - }

711 - }, {

712 - "type": "text",

713 - "text": "and this one"

714 - }, {

715 - "type": "image_url",

716 - "image_url": {

717 - "url": image_url

718 - }

719 - }, {

720 - "type": "text",

721 - "text": "Do they have differences?"

722 - }]

776 + "content": [

777 + {

778 + "type": "text",

779 + "text": "I need you to compare this image",

780 + },

781 + {

782 + "type": "image_url",

783 + "image_url": {

784 + "url": image_url

785 + }

786 + },

787 + {

788 + "type": "text",

789 + "text": "and this one"

790 + },

791 + {

792 + "type": "image_url",

793 + "image_url": {

794 + "url": image_url

795 + }

796 + },

797 + {

798 + "type": "text",

799 + "text": "Do they have differences?"

800 + },

801 + ],

802 }],

803 phi3v_model_config_mm_interleaved,

804 phi3v_tokenizer,

@@ -731,7 +810,7 @@ def test_parse_chat_messages_multiple_images_interleave(

810 "user",

811 "content":

812 "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501

734 - "Do they have differences?"

813 + "Do they have differences?",

814 }]

815 _assert_mm_data_is_image_input(mm_data, 2)

816

@@ -746,26 +825,32 @@ async def test_parse_chat_messages_multiple_images_interleave_async(

825 [{

826 "role":

827 "user",

749 - "content": [{

750 - "type": "text",

751 - "text": "I need you to compare this image"

752 - }, {

753 - "type": "image_url",

754 - "image_url": {

755 - "url": image_url

756 - }

757 - }, {

758 - "type": "text",

759 - "text": "and this one"

760 - }, {

761 - "type": "image_url",

762 - "image_url": {

763 - "url": image_url

764 - }

765 - }, {

766 - "type": "text",

767 - "text": "Do they have differences?"

768 - }]

828 + "content": [

829 + {

830 + "type": "text",

831 + "text": "I need you to compare this image",

832 + },

833 + {

834 + "type": "image_url",

835 + "image_url": {

836 + "url": image_url

837 + }

838 + },

839 + {

840 + "type": "text",

841 + "text": "and this one"

842 + },

843 + {

844 + "type": "image_url",

845 + "image_url": {

846 + "url": image_url

847 + }

848 + },

849 + {

850 + "type": "text",

851 + "text": "Do they have differences?"

852 + },

853 + ],

854 }],

855 phi3v_model_config_mm_interleaved,

856 phi3v_tokenizer,

@@ -777,7 +862,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(

862 "user",

863 "content":

864 "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501

780 - "Do they have differences?"

865 + "Do they have differences?",

866 }]

867 _assert_mm_data_is_image_input(await mm_data, 2)

868

@@ -788,135 +873,161 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(

873 image_url,

874 ):

875 conversation, mm_data = parse_chat_messages(

791 - [{

792 - "role":

793 - "user",

794 - "content": [

795 - {

796 - "type": "text",

797 - "text": "What's on this image?"

798 - },

799 - {

800 - "type": "image_url",

801 - "image_url": {

802 - "url": image_url

803 - }

804 - },

805 - {

806 - "type": "text",

807 - "text": "Be accurate."

808 - },

809 - ]

810 - }, {

811 - "role": "assistant",

812 - "content": "Some stuff."

813 - }, {

814 - "role":

815 - "user",

816 - "content": [{

817 - "type": "text",

818 - "text": "What's on this image?"

819 - }, {

820 - "type": "image_url",

821 - "image_url": {

822 - "url": image_url

823 - }

824 - }]

825 - }],

876 + [

877 + {

878 + "role":

879 + "user",

880 + "content": [

881 + {

882 + "type": "text",

883 + "text": "What's on this image?"

884 + },

885 + {

886 + "type": "image_url",

887 + "image_url": {

888 + "url": image_url

889 + }

890 + },

891 + {

892 + "type": "text",

893 + "text": "Be accurate."

894 + },

895 + ],

896 + },

897 + {

898 + "role": "assistant",

899 + "content": "Some stuff."

900 + },

901 + {

902 + "role":

903 + "user",

904 + "content": [

905 + {

906 + "type": "text",

907 + "text": "What's on this image?"

908 + },

909 + {

910 + "type": "image_url",

911 + "image_url": {

912 + "url": image_url

913 + }

914 + },

915 + ],

916 + },

917 + ],

918 phi3v_model_config_mm_interleaved,

919 phi3v_tokenizer,

920 content_format="string",

921 )

922

831 - assert conversation == [{

832 - "role":

833 - "user",

834 - "content":

835 - "What's on this image?\n<|image_1|>\nBe accurate."

836 - }, {

837 - "role": "assistant",

838 - "content": "Some stuff."

839 - }, {

840 - "role": "user",

841 - "content": "What's on this image?\n<|image_2|>"

842 - }]

923 + assert conversation == [

924 + {

925 + "role": "user",

926 + "content": "What's on this image?\n<|image_1|>\nBe accurate.",

927 + },

928 + {

929 + "role": "assistant",

930 + "content": "Some stuff."

931 + },

932 + {

933 + "role": "user",

934 + "content": "What's on this image?\n<|image_2|>"

935 + },

936 + ]

937 _assert_mm_data_is_image_input(mm_data, 2)

938

939

940 def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(

847 - qwen25omni_model_config_mm_interleaved, qwen25omni_tokenizer,

848 - image_url, video_url, audio_url):

941 + qwen25omni_model_config_mm_interleaved,

942 + qwen25omni_tokenizer,

943 + image_url,

944 + video_url,

945 + audio_url,

946 +):

947 conversation, mm_data = parse_chat_messages(

850 - [{

948 + [

949 + {

950 + "role":

951 + "user",

952 + "content": [

953 + {

954 + "type": "text",

955 + "text": "What's on this image?"

956 + },

957 + {

958 + "type": "image_url",

959 + "image_url": {

960 + "url": image_url

961 + }

962 + },

963 + {

964 + "type": "text",

965 + "text": "Now listen to this audio"

966 + },

967 + {

968 + "type": "audio_url",

969 + "audio_url": {

970 + "url": audio_url

971 + }

972 + },

973 + ],

974 + },

975 + {

976 + "role": "assistant",

977 + "content": "Some stuff."

978 + },

979 + {

980 + "role":

981 + "user",

982 + "content": [

983 + {

984 + "type": "text",

985 + "text": "What's on this image?"

986 + },

987 + {

988 + "type": "image_url",

989 + "image_url": {

990 + "url": image_url

991 + }

992 + },

993 + {

994 + "type": "text",

995 + "text": "And what's in the video?"

996 + },

997 + {

998 + "type": "video_url",

999 + "video_url": {

1000 + "url": video_url

1001 + }

1002 + },

1003 + ],

1004 + },

1005 + ],

1006 + qwen25omni_model_config_mm_interleaved,

1007 + qwen25omni_tokenizer,

1008 + content_format="string",

1009 + )

1010 +

1011 + assert conversation == [

1012 + {

1013 "role":

1014 "user",

853 - "content": [

854 - {

855 - "type": "text",

856 - "text": "What's on this image?"

857 - },

858 - {

859 - "type": "image_url",

860 - "image_url": {

861 - "url": image_url

862 - }

863 - },

864 - {

865 - "type": "text",

866 - "text": "Now listen to this audio"

867 - },

868 - {

869 - "type": "audio_url",

870 - "audio_url": {

871 - "url": audio_url

872 - }

873 - },

874 - ]

875 - }, {

1015 + "content":

1016 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"

1018 + },

1019 + {

1020 "role": "assistant",

1021 "content": "Some stuff."

878 - }, {

1022 + },

1023 + {

1024 "role":

1025 "user",

881 - "content": [{

882 - "type": "text",

883 - "text": "What's on this image?"

884 - }, {

885 - "type": "image_url",

886 - "image_url": {

887 - "url": image_url

888 - }

889 - }, {

890 - "type": "text",

891 - "text": "And what's in the video?"

892 - }, {

893 - "type": "video_url",

894 - "video_url": {

895 - "url": video_url

896 - }

897 - }]

898 - }],

899 - qwen25omni_model_config_mm_interleaved,

900 - qwen25omni_tokenizer,

901 - content_format="string",

902 - )

903 -

904 - assert conversation == [{

905 - "role":

906 - "user",

907 - "content":

908 - "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"

909 - "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>"

910 - }, {

911 - "role": "assistant",

912 - "content": "Some stuff."

913 - }, {

914 - "role":

915 - "user",

916 - "content":

917 - "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"

918 - "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>"

919 - }]

1026 + "content":

1027 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"

1028 + "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",

1029 + },

1030 + ]

1031

1032 _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})

1033

@@ -929,7 +1040,8 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(

1040 with pytest.raises(

1041 ValueError,

1042 match=r"Found more '<|image_1|>' placeholders in input prompt "

932 - "than actual multimodal data items."):

1043 + "than actual multimodal data items.",

1044 + ):

1045 parse_chat_messages(

1046 [{

1047 "role":

@@ -952,9 +1064,9 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(

1064 "text",

1065 "text":

1066 "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501

955 - "Do they have differences?"

1067 + "Do they have differences?",

1068 },

957 - ]

1069 + ],

1070 }],

1071 phi3v_model_config_mm_interleaved,

1072 phi3v_tokenizer,

@@ -973,12 +1085,15 @@ def test_mllama_single_image(

1085 [{

1086 "role":

1087 "user",

976 - "content": [{

977 - 'type': 'text',

978 - 'text': 'The content of this image is:'

979 - }, {

980 - "image_url": image_url

981 - }]

1088 + "content": [

1089 + {

1090 + "type": "text",

1091 + "text": "The content of this image is:"

1092 + },

1093 + {

1094 + "image_url": image_url

1095 + },

1096 + ],

1097 }],

1098 mllama_model_config,

1099 mllama_tokenizer,

@@ -986,14 +1101,17 @@ def test_mllama_single_image(

1101 )

1102 _assert_mm_data_is_image_input(mm_data, 1)

1103 assert conversation == [{

989 - 'role':

990 - 'user',

991 - 'content': [{

992 - 'type': 'text',

993 - 'text': 'The content of this image is:'

994 - }, {

995 - 'type': 'image'

996 - }]

1104 + "role":

1105 + "user",

1106 + "content": [

1107 + {

1108 + "type": "text",

1109 + "text": "The content of this image is:"

1110 + },

1111 + {

1112 + "type": "image"

1113 + },

1114 + ],

1115 }]

1116

1117

@@ -1009,20 +1127,20 @@ def test_mllama_interleaved_images(

1127 "user",

1128 "content": [

1129 {

1012 - 'type': 'text',

1013 - 'text': 'The content of the first image is:'

1130 + "type": "text",

1131 + "text": "The content of the first image is:",

1132 },

1133 {

1134 "image_url": image_url

1135 },

1136 {

1019 - 'type': 'text',

1020 - 'text': 'The content of the second image is:'

1137 + "type": "text",

1138 + "text": "The content of the second image is:",

1139 },

1140 {

1141 "image_url": image_url

1142 },

1025 - ]

1143 + ],

1144 }],

1145 mllama_model_config,

1146 mllama_tokenizer,

@@ -1030,19 +1148,24 @@ def test_mllama_interleaved_images(

1148 )

1149 _assert_mm_data_is_image_input(mm_data, 2)

1150 assert conversation == [{

1033 - 'role':

1034 - 'user',

1035 - 'content': [{

1036 - 'type': 'text',

1037 - 'text': 'The content of the first image is:'

1038 - }, {

1039 - 'type': 'image'

1040 - }, {

1041 - 'type': 'text',

1042 - 'text': 'The content of the second image is:'

1043 - }, {

1044 - 'type': 'image'

1045 - }]

1151 + "role":

1152 + "user",

1153 + "content": [

1154 + {

1155 + "type": "text",

1156 + "text": "The content of the first image is:"

1157 + },

1158 + {

1159 + "type": "image"

1160 + },

1161 + {

1162 + "type": "text",

1163 + "text": "The content of the second image is:"

1164 + },

1165 + {

1166 + "type": "image"

1167 + },

1168 + ],

1169 }]

1170

1171

@@ -1053,34 +1176,36 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):

1176 def get_conversation(is_hf: bool):

1177 img_part = {"type": "image_url", "image_url": {"url": image_url}}

1178 if is_hf:

1056 - img_part = {'type': 'image'}

1179 + img_part = {"type": "image"}

1180 return [{

1058 - 'role':

1059 - 'user',

1060 - 'content': [

1181 + "role":

1182 + "user",

1183 + "content": [

1184 {

1062 - 'type': 'text',

1063 - 'text': 'The content of the first image is:'

1185 + "type": "text",

1186 + "text": "The content of the first image is:",

1187 },

1188 img_part,

1189 {

1067 - 'type': 'text',

1068 - 'text': 'The content of the second image is:'

1190 + "type": "text",

1191 + "text": "The content of the second image is:",

1192 },

1193 img_part,

1194 {

1072 - 'type': 'text',

1073 - 'text': 'What animal is in the first image?'

1195 + "type": "text",

1196 + "text": "What animal is in the first image?",

1197 },

1075 - ]

1198 + ],

1199 }]

1200

1201 # Build a config for the model

1079 - model_config = ModelConfig(model,

1080 - runner="generate",

1081 - limit_mm_per_prompt={

1082 - "image": 2,

1083 - })

1202 + model_config = ModelConfig(

1203 + model,

1204 + runner="generate",

1205 + limit_mm_per_prompt={

1206 + "image": 2,

1207 + },

1208 + )

1209

1210 # Build the tokenizer group and grab the underlying tokenizer

1211 tokenizer_group = TokenizerGroup(

@@ -1126,7 +1251,8 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):

1251 [

1252 QWEN2VL_MODEL_ID, # tokenizer.chat_template is of type str

1253 HERMES_MODEL_ID, # tokenizer.chat_template is of type dict

1129 - ])

1254 + ],

1255 +)

1256 @pytest.mark.parametrize("use_tools", [True, False])

1257 def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

1258 """checks that chat_template is a dict type for HF models."""

@@ -1152,14 +1278,14 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

1278 )

1279 tokenizer = tokenizer_group.tokenizer

1280

1155 - tools = [{

1281 + tools = ([{

1282 "type": "function",

1283 "function": {

1284 "name": "dummy_function_name",

1285 "description": "This is a dummy function",

1160 - "parameters": sample_json_schema

1161 - }

1162 - }] if use_tools else None

1286 + "parameters": sample_json_schema,

1287 + },

1288 + }] if use_tools else None)

1289

1290 # Test detecting the tokenizer's chat_template

1291 chat_template = resolve_hf_chat_template(

vllm/entrypoints/chat_utils.py

+228 -150

@@ -103,6 +103,7 @@ class PILImage(BaseModel):

103 """

104 A PIL.Image.Image object.

105 """

106 +

107 image_pil: Image.Image

108 model_config = ConfigDict(arbitrary_types_allowed=True)

109

@@ -115,6 +116,7 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):

116 "image_pil": ImageAsset('cherry_blossom').pil_image

117 }

118 """

119 +

120 image_pil: Required[PILImage]

121

122

@@ -127,6 +129,7 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):

129 "image_url": "https://example.com/image.jpg"

130 }

131 """

132 +

133 image_url: Required[str]

134

135

@@ -138,6 +141,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):

141 "audio_url": "https://example.com/audio.mp3"

142 }

143 """

144 +

145 audio_url: Required[str]

146

147

@@ -149,6 +153,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):

153 "video_url": "https://example.com/video.mp4"

154 }

155 """

156 +

157 video_url: Required[str]

158

159

@@ -174,19 +179,24 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):

179

180

181 ChatCompletionContentPartParam: TypeAlias = Union[

177 - OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,

182 + OpenAIChatCompletionContentPartParam,

183 + ChatCompletionContentPartAudioParam,

184 ChatCompletionContentPartInputAudioParam,

179 - ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,

185 + ChatCompletionContentPartVideoParam,

186 + ChatCompletionContentPartRefusalParam,

187 CustomChatCompletionContentPILImageParam,

188 CustomChatCompletionContentSimpleImageParam,

189 ChatCompletionContentPartImageEmbedsParam,

190 CustomChatCompletionContentSimpleAudioParam,

184 - CustomChatCompletionContentSimpleVideoParam, str,

185 - CustomThinkCompletionContentParam]

191 + CustomChatCompletionContentSimpleVideoParam,

192 + str,

193 + CustomThinkCompletionContentParam,

194 +]

195

196

197 class CustomChatCompletionMessageParam(TypedDict, total=False):

198 """Enables custom roles in the Chat Completion API."""

199 +

200 role: Required[str]

201 """The role of the message's author."""

202

@@ -207,9 +217,11 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):

217 """The tool calls generated by the model, such as function calls."""

218

219

210 -ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,

211 - CustomChatCompletionMessageParam,

212 - OpenAIHarmonyMessage]

220 +ChatCompletionMessageParam = Union[

221 + OpenAIChatCompletionMessageParam,

222 + CustomChatCompletionMessageParam,

223 + OpenAIHarmonyMessage,

224 +]

225

226

227 # TODO: Make fields ReadOnly once mypy supports it

@@ -262,13 +274,13 @@ def _is_var_or_elems_access(

274 key: Optional[str] = None,

275 ) -> bool:

276 if isinstance(node, jinja2.nodes.Filter):

265 - return (node.node is not None

266 - and _is_var_or_elems_access(node.node, varname, key))

277 + return node.node is not None and _is_var_or_elems_access(

278 + node.node, varname, key)

279 if isinstance(node, jinja2.nodes.Test):

280 return _is_var_or_elems_access(node.node, varname, key)

281

270 - if (isinstance(node, jinja2.nodes.Getitem)

271 - and isinstance(node.arg, jinja2.nodes.Slice)):

282 + if isinstance(node, jinja2.nodes.Getitem) and isinstance(

283 + node.arg, jinja2.nodes.Slice):

284 return _is_var_or_elems_access(node.node, varname, key)

285

286 # yapf: disable

@@ -373,15 +385,18 @@ def resolve_mistral_chat_template(

385 ) -> Optional[str]:

386 if chat_template is not None:

387 logger.warning_once(

376 - "'chat_template' cannot be overridden for mistral tokenizer.")

388 + "'chat_template' cannot be overridden for mistral tokenizer."

389 + )

390 if "add_generation_prompt" in kwargs:

391 logger.warning_once(

392 "'add_generation_prompt' is not supported for mistral tokenizer, "

380 - "so it will be ignored.")

393 + "so it will be ignored."

394 + )

395 if "continue_final_message" in kwargs:

396 logger.warning_once(

397 "'continue_final_message' is not supported for mistral tokenizer, "

384 - "so it will be ignored.")

398 + "so it will be ignored."

399 + )

400 return None

401

402

@@ -401,23 +416,35 @@ def resolve_hf_chat_template(

416 try:

417 processor = cached_get_processor(

418 tokenizer.name_or_path,

404 - processor_cls=(PreTrainedTokenizer, PreTrainedTokenizerFast,

405 - ProcessorMixin),

419 + processor_cls=(

420 + PreTrainedTokenizer,

421 + PreTrainedTokenizerFast,

422 + ProcessorMixin,

423 + ),

424 trust_remote_code=model_config.trust_remote_code,

425 )

408 - if isinstance(processor, ProcessorMixin) and \

409 - hasattr(processor, 'chat_template') and \

410 - processor.chat_template is not None:

426 + if (

427 + isinstance(processor, ProcessorMixin)

428 + and hasattr(processor, "chat_template")

429 + and processor.chat_template is not None

430 + ):

431 return processor.chat_template

432 except Exception:

413 - logger.debug("Failed to load AutoProcessor chat template for %s", tokenizer.name_or_path, exc_info=True) # noqa: E501

433 + logger.debug(

434 + "Failed to load AutoProcessor chat template for %s",

435 + tokenizer.name_or_path,

436 + exc_info=True,

437 + ) # noqa: E501

438

439 # 3rd priority: AutoTokenizer chat template

440 try:

441 return tokenizer.get_chat_template(chat_template, tools=tools)

442 except Exception:

419 - logger.debug("Failed to load AutoTokenizer chat template for %s",

420 - tokenizer.name_or_path, exc_info=True)

443 + logger.debug(

444 + "Failed to load AutoTokenizer chat template for %s",

445 + tokenizer.name_or_path,

446 + exc_info=True,

447 + )

448

449 # 4th priority: Predefined fallbacks

450 path = get_chat_template_fallback_path(

@@ -425,12 +452,16 @@ def resolve_hf_chat_template(

452 tokenizer_name_or_path=model_config.tokenizer,

453 )

454 if path is not None:

428 - logger.info("Loading chat template fallback for %s as there isn't one "

429 - "defined on HF Hub.", tokenizer.name_or_path)

455 + logger.info(

456 + "Loading chat template fallback for %s as there isn't one "

457 + "defined on HF Hub.",

458 + tokenizer.name_or_path,

459 + )

460 chat_template = load_chat_template(path)

461 else:

432 - logger.debug("There is no chat template fallback for %s",

433 - tokenizer.name_or_path)

462 + logger.debug(

463 + "There is no chat template fallback for %s", tokenizer.name_or_path

464 + )

465

466 return chat_template

467

@@ -452,11 +483,17 @@ def _resolve_chat_template_content_format(

483 else:

484 hf_chat_template = None

485

455 - jinja_text = (hf_chat_template if isinstance(hf_chat_template, str)

456 - else load_chat_template(chat_template, is_literal=True))

486 + jinja_text = (

487 + hf_chat_template

488 + if isinstance(hf_chat_template, str)

489 + else load_chat_template(chat_template, is_literal=True)

490 + )

491

458 - detected_format = ("string" if jinja_text is None else

459 - _detect_content_format(jinja_text, default="string"))

492 + detected_format = (

493 + "string"

494 + if jinja_text is None

495 + else _detect_content_format(jinja_text, default="string")

496 + )

497

498 return detected_format

499

@@ -512,7 +549,6 @@ def resolve_chat_template_content_format(

549 return detected_format

550

551

515 -

552 ModalityStr = Literal["image", "audio", "video", "image_embeds"]

553 _T = TypeVar("_T")

554

@@ -539,6 +575,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

575 @cached_property

576 def model_cls(self) -> type[SupportsMultiModal]:

577 from vllm.model_executor.model_loader import get_model_cls

578 +

579 model_cls = get_model_cls(self.model_config)

580 return cast(type[SupportsMultiModal], model_cls)

581

@@ -574,28 +611,29 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

611

612

613 class MultiModalItemTracker(BaseMultiModalItemTracker[object]):

577 -

614 def all_mm_data(self) -> Optional[MultiModalDataDict]:

615 if not self._items_by_modality:

616 return None

617 mm_inputs = {}

618 items_by_modality = dict(self._items_by_modality)

619 if "image" in items_by_modality and "image_embeds" in items_by_modality:

584 - raise ValueError(\

585 - "Mixing raw image and embedding inputs is not allowed")

620 + raise ValueError(

621 + "Mixing raw image and embedding inputs is not allowed"

622 + )

623

624 if "image_embeds" in items_by_modality:

625 image_embeds_lst = items_by_modality["image_embeds"]

626 if len(image_embeds_lst) > 1:

590 - raise ValueError(\

591 - "Only one message can have {'type': 'image_embeds'}")

627 + raise ValueError(

628 + "Only one message can have {'type': 'image_embeds'}"

629 + )

630 mm_inputs["image"] = image_embeds_lst[0]

631 if "image" in items_by_modality:

594 - mm_inputs["image"] = items_by_modality["image"] # A list of images

632 + mm_inputs["image"] = items_by_modality["image"] # A list of images

633 if "audio" in items_by_modality:

596 - mm_inputs["audio"] = items_by_modality["audio"] # A list of audios

634 + mm_inputs["audio"] = items_by_modality["audio"] # A list of audios

635 if "video" in items_by_modality:

598 - mm_inputs["video"] = items_by_modality["video"] # A list of videos

636 + mm_inputs["video"] = items_by_modality["video"] # A list of videos

637 return mm_inputs

638

639 def create_parser(self) -> "BaseMultiModalContentParser":

@@ -603,32 +641,33 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[object]):

641

642

643 class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):

606 -

644 async def all_mm_data(self) -> Optional[MultiModalDataDict]:

645 if not self._items_by_modality:

646 return None

647 mm_inputs = {}

648 items_by_modality = {

612 - modality: await asyncio.gather(*items)

613 - for modality, items in self._items_by_modality.items()

614 - }

649 + modality: await asyncio.gather(*items)

650 + for modality, items in self._items_by_modality.items()

651 + }

652

653 if "image" in items_by_modality and "image_embeds" in items_by_modality:

654 raise ValueError(

618 - "Mixing raw image and embedding inputs is not allowed")

655 + "Mixing raw image and embedding inputs is not allowed"

656 + )

657

658 if "image_embeds" in items_by_modality:

659 image_embeds_lst = items_by_modality["image_embeds"]

660 if len(image_embeds_lst) > 1:

661 raise ValueError(

624 - "Only one message can have {'type': 'image_embeds'}")

662 + "Only one message can have {'type': 'image_embeds'}"

663 + )

664 mm_inputs["image"] = image_embeds_lst[0]

665 if "image" in items_by_modality:

627 - mm_inputs["image"] = items_by_modality["image"] # A list of images

666 + mm_inputs["image"] = items_by_modality["image"] # A list of images

667 if "audio" in items_by_modality:

629 - mm_inputs["audio"] = items_by_modality["audio"] # A list of audios

668 + mm_inputs["audio"] = items_by_modality["audio"] # A list of audios

669 if "video" in items_by_modality:

631 - mm_inputs["video"] = items_by_modality["video"] # A list of videos

670 + mm_inputs["video"] = items_by_modality["video"] # A list of videos

671 return mm_inputs

672

673 def create_parser(self) -> "BaseMultiModalContentParser":

@@ -636,7 +675,6 @@ class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):

675

676

677 class BaseMultiModalContentParser(ABC):

639 -

678 def __init__(self) -> None:

679 super().__init__()

680

@@ -648,8 +686,9 @@ class BaseMultiModalContentParser(ABC):

686 # }

687 self._placeholder_storage: dict[str, list] = defaultdict(list)

688

651 - def _add_placeholder(self, modality: ModalityStr,

652 - placeholder: Optional[str]):

689 + def _add_placeholder(

690 + self, modality: ModalityStr, placeholder: Optional[str]

691 + ):

692 mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]

693 if placeholder:

694 self._placeholder_storage[mod_placeholder].append(placeholder)

@@ -662,8 +701,9 @@ class BaseMultiModalContentParser(ABC):

701 raise NotImplementedError

702

703 @abstractmethod

665 - def parse_image_embeds(self,

666 - image_embeds: Union[str, dict[str, str]]) -> None:

704 + def parse_image_embeds(

705 + self, image_embeds: Union[str, dict[str, str]]

706 + ) -> None:

707 raise NotImplementedError

708

709 @abstractmethod

@@ -684,7 +724,6 @@ class BaseMultiModalContentParser(ABC):

724

725

726 class MultiModalContentParser(BaseMultiModalContentParser):

687 -

727 def __init__(self, tracker: MultiModalItemTracker) -> None:

728 super().__init__()

729

@@ -701,8 +740,9 @@ class MultiModalContentParser(BaseMultiModalContentParser):

740 placeholder = self._tracker.add("image", image)

741 self._add_placeholder("image", placeholder)

742

704 - def parse_image_embeds(self,

705 - image_embeds: Union[str, dict[str, str]]) -> None:

743 + def parse_image_embeds(

744 + self, image_embeds: Union[str, dict[str, str]]

745 + ) -> None:

746 if isinstance(image_embeds, dict):

747 embeds = {

748 k: self._connector.fetch_image_embedding(v)

@@ -741,14 +781,13 @@ class MultiModalContentParser(BaseMultiModalContentParser):

781

782

783 class AsyncMultiModalContentParser(BaseMultiModalContentParser):

744 -

784 def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:

785 super().__init__()

786

787 self._tracker = tracker

788 self._connector = MediaConnector(

789 media_io_kwargs=self._tracker._model_config.media_io_kwargs,

751 - allowed_local_media_path=tracker.allowed_local_media_path

790 + allowed_local_media_path=tracker.allowed_local_media_path,

791 )

792

793 def parse_image(self, image_url: str) -> None:

@@ -757,8 +796,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

796 placeholder = self._tracker.add("image", image_coro)

797 self._add_placeholder("image", placeholder)

798

760 - def parse_image_embeds(self,

761 - image_embeds: Union[str, dict[str, str]]) -> None:

799 + def parse_image_embeds(

800 + self, image_embeds: Union[str, dict[str, str]]

801 + ) -> None:

802 future: asyncio.Future[Union[str, dict[str, str]]] = asyncio.Future()

803

804 if isinstance(image_embeds, dict):

@@ -769,8 +809,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

809 future.set_result(embeds)

810

811 if isinstance(image_embeds, str):

772 - embedding = self._connector.\

773 - fetch_image_embedding(image_embeds)

812 + embedding = self._connector.fetch_image_embedding(image_embeds)

813 future.set_result(embedding)

814

815 placeholder = self._tracker.add("image_embeds", future)

@@ -809,20 +848,23 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):

848 return

849

850 elif isinstance(chat_template, Path) and not chat_template.exists():

812 - raise FileNotFoundError(

813 - "the supplied chat template path doesn't exist")

851 + raise FileNotFoundError("the supplied chat template path doesn't exist")

852

853 elif isinstance(chat_template, str):

854 JINJA_CHARS = "{}\n"

817 - if not any(c in chat_template

818 - for c in JINJA_CHARS) and not Path(chat_template).exists():

855 + if (

856 + not any(c in chat_template for c in JINJA_CHARS)

857 + and not Path(chat_template).exists()

858 + ):

859 raise ValueError(

860 f"The supplied chat template string ({chat_template}) "

821 - f"appears path-like, but doesn't exist!")

861 + f"appears path-like, but doesn't exist!"

862 + )

863

864 else:

865 raise TypeError(

825 - f"{type(chat_template)} is not a valid chat template type")

866 + f"{type(chat_template)} is not a valid chat template type"

867 + )

868

869

870 def _load_chat_template(

@@ -835,8 +877,9 @@ def _load_chat_template(

877

878 if is_literal:

879 if isinstance(chat_template, Path):

838 - raise TypeError("chat_template is expected to be read directly "

839 - "from its value")

880 + raise TypeError(

881 + "chat_template is expected to be read directly from its value"

882 + )

883

884 return chat_template

885

@@ -849,9 +892,11 @@ def _load_chat_template(

892

893 JINJA_CHARS = "{}\n"

894 if not any(c in chat_template for c in JINJA_CHARS):

852 - msg = (f"The supplied chat template ({chat_template}) "

853 - f"looks like a file path, but it failed to be "

854 - f"opened. Reason: {e}")

895 + msg = (

896 + f"The supplied chat template ({chat_template}) "

897 + f"looks like a file path, but it failed to be "

898 + f"opened. Reason: {e}"

899 + )

900 raise ValueError(msg) from e

901

902 # If opening a file fails, set chat template to be args to

@@ -870,8 +915,9 @@ def load_chat_template(

915 return _cached_load_chat_template(chat_template, is_literal=is_literal)

916

917

873 -def _get_interleaved_text_prompt(placeholder_storage: dict[str, list],

874 - texts: list[str]) -> str:

918 +def _get_interleaved_text_prompt(

919 + placeholder_storage: dict[str, list], texts: list[str]

920 +) -> str:

921 for idx, elem in enumerate(texts):

922 if elem in placeholder_storage:

923 texts[idx] = placeholder_storage[elem].pop(0)

@@ -881,10 +927,11 @@ def _get_interleaved_text_prompt(placeholder_storage: dict[str, list],

927

928 # TODO: Let user specify how to insert multimodal tokens into prompt

929 # (similar to chat template)

884 -def _get_full_multimodal_text_prompt(placeholder_storage: dict[str, list],

885 - texts: list[str],

886 - interleave_strings: bool

887 - ) -> str:

930 +def _get_full_multimodal_text_prompt(

931 + placeholder_storage: dict[str, list],

932 + texts: list[str],

933 + interleave_strings: bool,

934 +) -> str:

935 """Combine multimodal prompts for a multimodal language model."""

936

937 # flatten storage to make it looks like

@@ -907,7 +954,6 @@ def _get_full_multimodal_text_prompt(placeholder_storage: dict[str, list],

954 # Look through the text prompt to check for missing placeholders

955 missing_placeholders: list[str] = []

956 for placeholder in placeholder_counts:

910 -

957 # For any existing placeholder in the text prompt, we leave it as is

958 placeholder_counts[placeholder] -= text_prompt.count(placeholder)

959

@@ -916,15 +962,18 @@ def _get_full_multimodal_text_prompt(placeholder_storage: dict[str, list],

962 "Placeholder count is negative! "

963 "Ensure that the 'interleave_strings' flag is disabled "

964 "(current value: %s) "

919 - "when manually placing image placeholders.", interleave_strings

965 + "when manually placing image placeholders.",

966 + interleave_strings,

967 )

968 logger.debug("Input prompt: %s", text_prompt)

969 raise ValueError(

970 f"Found more '{placeholder}' placeholders in input prompt than "

924 - "actual multimodal data items.")

971 + "actual multimodal data items."

972 + )

973

926 - missing_placeholders.extend([placeholder] *

927 - placeholder_counts[placeholder])

974 + missing_placeholders.extend(

975 + [placeholder] * placeholder_counts[placeholder]

976 + )

977

978 # NOTE: Default behaviour: we always add missing placeholders

979 # at the front of the prompt, if interleave_strings=False

@@ -944,7 +993,8 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python

993 _VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python

994

995 _ResponsesInputImageParser = TypeAdapter(

947 - ResponseInputImageParam).validate_python

996 + ResponseInputImageParam

997 +).validate_python

998 _ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio, PILImage]

999

1000 # Define a mapping from part types to their corresponding parsing functions.

@@ -952,32 +1002,35 @@ MM_PARSER_MAP: dict[

1002 str,

1003 Callable[[ChatCompletionContentPartParam], _ContentPart],

1004 ] = {

955 - "text":

956 - lambda part: _TextParser(part).get("text", None),

957 - "thinking":

958 - lambda part: _ThinkParser(part).get("thinking", None),

959 - "input_text":

960 - lambda part: _TextParser(part).get("text", None),

961 - "input_image":

962 - lambda part: _ResponsesInputImageParser(part).get("image_url", None),

963 - "image_url":

964 - lambda part: _ImageParser(part).get("image_url", {}).get("url", None),

965 - "image_embeds":

966 - lambda part: _ImageEmbedsParser(part).get("image_embeds", None),

1005 + "text": lambda part: _TextParser(part).get("text", None),

1006 + "thinking": lambda part: _ThinkParser(part).get("thinking", None),

1007 + "input_text": lambda part: _TextParser(part).get("text", None),

1008 + "input_image": lambda part: _ResponsesInputImageParser(part).get(

1009 + "image_url", None

1010 + ),

1011 + "image_url": lambda part: _ImageParser(part)

1012 + .get("image_url", {})

1013 + .get("url", None),

1014 + "image_embeds": lambda part: _ImageEmbedsParser(part).get(

1015 + "image_embeds", None

1016 + ),

1017 "image_pil": lambda part: _PILImageParser(part).get("image_pil", None),

968 - "audio_url":

969 - lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),

970 - "input_audio":

971 - lambda part: _InputAudioParser(part).get("input_audio", None),

972 - "refusal":

973 - lambda part: _RefusalParser(part).get("refusal", None),

974 - "video_url":

975 - lambda part: _VideoParser(part).get("video_url", {}).get("url", None),

1018 + "audio_url": lambda part: _AudioParser(part)

1019 + .get("audio_url", {})

1020 + .get("url", None),

1021 + "input_audio": lambda part: _InputAudioParser(part).get(

1022 + "input_audio", None

1023 + ),

1024 + "refusal": lambda part: _RefusalParser(part).get("refusal", None),

1025 + "video_url": lambda part: _VideoParser(part)

1026 + .get("video_url", {})

1027 + .get("url", None),

1028 }

1029

1030

1031 def _parse_chat_message_content_mm_part(

980 - part: ChatCompletionContentPartParam) -> tuple[str, _ContentPart]:

1032 + part: ChatCompletionContentPartParam,

1033 +) -> tuple[str, _ContentPart]:

1034 """

1035 Parses a given multi-modal content part based on its type.

1036

@@ -993,7 +1046,8 @@ def _parse_chat_message_content_mm_part(

1046 ValueError: If the 'type' field is missing and no direct URL is found.

1047 """

1048 assert isinstance(

996 - part, dict) # This is needed to avoid mypy errors: part.get() from str

1049 + part, dict

1050 + ) # This is needed to avoid mypy errors: part.get() from str

1051 part_type = part.get("type", None)

1052

1053 if isinstance(part_type, str) and part_type in MM_PARSER_MAP:

@@ -1002,8 +1056,10 @@ def _parse_chat_message_content_mm_part(

1056 # Special case for 'image_url.detail'

1057 # We only support 'auto', which is the default

1058 if part_type == "image_url" and part.get("detail", "auto") != "auto":

1005 - logger.warning("'image_url.detail' is currently not supported "

1006 - "and will be ignored.")

1059 + logger.warning(

1060 + "'image_url.detail' is currently not supported "

1061 + "and will be ignored."

1062 + )

1063

1064 return part_type, content

1065

@@ -1011,19 +1067,22 @@ def _parse_chat_message_content_mm_part(

1067 # 'type' is required field by pydantic

1068 if part_type is None:

1069 if part.get("image_url") is not None:

1014 - image_params = cast(CustomChatCompletionContentSimpleImageParam,

1015 - part)

1070 + image_params = cast(

1071 + CustomChatCompletionContentSimpleImageParam, part

1072 + )

1073 return "image_url", image_params.get("image_url", "")

1074 if part.get("audio_url") is not None:

1018 - audio_params = cast(CustomChatCompletionContentSimpleAudioParam,

1019 - part)

1075 + audio_params = cast(

1076 + CustomChatCompletionContentSimpleAudioParam, part

1077 + )

1078 return "audio_url", audio_params.get("audio_url", "")

1079 if part.get("input_audio") is not None:

1080 input_audio_params = cast(dict[str, str], part)

1081 return "input_audio", input_audio_params

1082 if part.get("video_url") is not None:

1025 - video_params = cast(CustomChatCompletionContentSimpleVideoParam,

1026 - part)

1083 + video_params = cast(

1084 + CustomChatCompletionContentSimpleVideoParam, part

1085 + )

1086 return "video_url", video_params.get("video_url", "")

1087 # Raise an error if no 'type' or direct URL is found.

1088 raise ValueError("Missing 'type' field in multimodal part.")

@@ -1033,9 +1092,16 @@ def _parse_chat_message_content_mm_part(

1092 return part_type, "unknown part_type content"

1093

1094

1036 -VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",

1037 - "image_embeds", "image_pil",

1038 - "audio_url", "input_audio", "video_url")

1095 +VALID_MESSAGE_CONTENT_MM_PART_TYPES = (

1096 + "text",

1097 + "refusal",

1098 + "image_url",

1099 + "image_embeds",

1100 + "image_pil",

1101 + "audio_url",

1102 + "input_audio",

1103 + "video_url",

1104 +)

1105

1106

1107 def _parse_chat_message_content_parts(

@@ -1055,21 +1121,20 @@ def _parse_chat_message_content_parts(

1121 part,

1122 mm_parser,

1123 wrap_dicts=wrap_dicts,

1058 - interleave_strings=interleave_strings

1124 + interleave_strings=interleave_strings,

1125 )

1126 if parse_res:

1127 content.append(parse_res)

1128

1129 if wrap_dicts:

1130 # Parsing wraps images and texts as interleaved dictionaries

1065 - return [ConversationMessage(role=role,

1066 - content=content)] # type: ignore

1131 + return [ConversationMessage(role=role, content=content)] # type: ignore

1132 texts = cast(list[str], content)

1133 mm_placeholder_storage = mm_parser.mm_placeholder_storage()

1134 if mm_placeholder_storage:

1070 - text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_storage,

1071 - texts,

1072 - interleave_strings)

1135 + text_prompt = _get_full_multimodal_text_prompt(

1136 + mm_placeholder_storage, texts, interleave_strings

1137 + )

1138 else:

1139 text_prompt = "\n".join(texts)

1140

@@ -1099,13 +1164,16 @@ def _parse_chat_message_content_part(

1164 if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:

1165 logger.warning(

1166 "Skipping multimodal part '%s' (type: '%s') "

1102 - "with empty / unparsable content.", part, part_type)

1167 + "with empty / unparsable content.",

1168 + part,

1169 + part_type,

1170 + )

1171 return None

1172

1173 if part_type in ("text", "input_text", "refusal", "thinking"):

1174 str_content = cast(str, content)

1175 if wrap_dicts:

1108 - return {'type': 'text', 'text': str_content}

1176 + return {"type": "text", "text": str_content}

1177 else:

1178 return str_content

1179

@@ -1137,8 +1205,12 @@ def _parse_chat_message_content_part(

1205 else:

1206 raise NotImplementedError(f"Unknown part type: {part_type}")

1207

1140 - return {'type': modality} if wrap_dicts else (

1141 - MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None

1208 + return (

1209 + {"type": modality}

1210 + if wrap_dicts

1211 + else (

1212 + MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None

1213 + )

1214 )

1215

1216

@@ -1171,14 +1243,16 @@ def _parse_chat_message_content(

1243 )

1244

1245 for result_msg in result:

1174 - if role == 'assistant':

1246 + if role == "assistant":

1247 parsed_msg = _AssistantParser(message)

1248

1249 # The 'tool_calls' is not None check ensures compatibility.

1250 # It's needed only if downstream code doesn't strictly

1251 # follow the OpenAI spec.

1180 - if ("tool_calls" in parsed_msg

1181 - and parsed_msg["tool_calls"] is not None):

1252 + if (

1253 + "tool_calls" in parsed_msg

1254 + and parsed_msg["tool_calls"] is not None

1255 + ):

1256 result_msg["tool_calls"] = list(parsed_msg["tool_calls"])

1257 elif role == "tool":

1258 parsed_msg = _ToolParser(message)

@@ -1198,12 +1272,15 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:

1272 # so, for messages that have tool_calls, parse the string (which we get

1273 # from openAI format) to dict

1274 for message in messages:

1201 - if (message["role"] == "assistant" and "tool_calls" in message

1202 - and isinstance(message["tool_calls"], list)):

1203 -

1275 + if (

1276 + message["role"] == "assistant"

1277 + and "tool_calls" in message

1278 + and isinstance(message["tool_calls"], list)

1279 + ):

1280 for item in message["tool_calls"]:

1281 item["function"]["arguments"] = json.loads(

1206 - item["function"]["arguments"])

1282 + item["function"]["arguments"]

1283 + )

1284

1285

1286 def parse_chat_messages(

@@ -1224,7 +1301,7 @@ def parse_chat_messages(

1301 content_format == "string"

1302 and model_config.multimodal_config is not None

1303 and model_config.multimodal_config.interleave_mm_strings

1227 - )

1304 + ),

1305 )

1306

1307 conversation.extend(sub_messages)

@@ -1252,7 +1329,7 @@ def parse_chat_messages_futures(

1329 content_format == "string"

1330 and model_config.multimodal_config is not None

1331 and model_config.multimodal_config.interleave_mm_strings

1255 - )

1332 + ),

1333 )

1334

1335 conversation.extend(sub_messages)

@@ -1283,10 +1360,10 @@ def apply_hf_chat_template(

1360 raise ValueError(

1361 "As of transformers v4.44, default chat template is no longer "

1362 "allowed, so you must provide a chat template if the tokenizer "

1286 - "does not define one.")

1363 + "does not define one."

1364 + )

1365

1366 try:

1289 -

1367 return tokenizer.apply_chat_template(

1368 conversation=conversation, # type: ignore[arg-type]

1369 tools=tools, # type: ignore[arg-type]

@@ -1298,13 +1375,14 @@ def apply_hf_chat_template(

1375 # External library exceptions can sometimes occur despite the framework's

1376 # internal exception management capabilities.

1377 except Exception as e:

1301 -

1378 # Log and report any library-related exceptions for further

1379 # investigation.

1380 logger.exception(

1305 - "An error occurred in `transformers` while applying chat template")

1381 + "An error occurred in `transformers` while applying chat template"

1382 + )

1383 raise ValueError(str(e)) from e

1384

1385 +

1386 def apply_mistral_chat_template(

1387 tokenizer: MistralTokenizer,

1388 messages: list[ChatCompletionMessageParam],

@@ -1337,26 +1415,26 @@ def apply_mistral_chat_template(

1415 # External library exceptions can sometimes occur despite the framework's

1416 # internal exception management capabilities.

1417 except Exception as e:

1340 -

1418 # Log and report any library-related exceptions for further

1419 # investigation.

1420 logger.exception(

1344 - "An error occurred in `mistral_common` while applying chat "

1345 - "template")

1421 + "An error occurred in `mistral_common` while applying chat template"

1422 + )

1423 raise ValueError(str(e)) from e

1424

1425 +

1426 def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):

1427 idx = 0

1428 for msg in conversation:

1351 - if msg['role'] == 'assistant':

1352 - tool_calls = msg.get('tool_calls')

1353 - idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa

1429 + if msg["role"] == "assistant":

1430 + tool_calls = msg.get("tool_calls")

1431 + idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa

1432 return idx

1433

1356 -def make_tool_call_id(id_type:str='random', func_name=None, idx=None):

1434

1358 - if id_type=='kimi_k2':

1359 - return f'functions.{func_name}:{idx}'

1435 +def make_tool_call_id(id_type: str = "random", func_name=None, idx=None):

1436 + if id_type == "kimi_k2":

1437 + return f"functions.{func_name}:{idx}"

1438 else:

1439 # by default return random

1440 return f"chatcmpl-tool-{random_uuid()}"

vllm/entrypoints/openai/serving_engine.py

+141 -88

@@ -82,16 +82,26 @@ from vllm.utils import (AsyncMicrobatchTokenizer, is_list_of,

83 logger = init_logger(__name__)

85 -CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,

86 - EmbeddingCompletionRequest, RerankRequest,

87 - ClassificationRequest, ScoreRequest,

88 - TokenizeCompletionRequest]

85 +CompletionLikeRequest = Union[

86 + CompletionRequest,

87 + DetokenizeRequest,

88 + EmbeddingCompletionRequest,

89 + RerankRequest,

90 + ClassificationRequest,

91 + ScoreRequest,

92 + TokenizeCompletionRequest,

93 +]

95 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,

96 TokenizeChatRequest]

97 SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]

93 -AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest, SpeechToTextRequest,

94 - ResponsesRequest, IOProcessorRequest]

98 +AnyRequest = Union[

99 + CompletionLikeRequest,

100 + ChatLikeRequest,

101 + SpeechToTextRequest,

102 + ResponsesRequest,

103 + IOProcessorRequest,

104 +]

105

106 AnyResponse = Union[

107 CompletionResponse,

@@ -135,6 +145,7 @@ class RequestProcessingMixin(BaseModel):

145 Mixin for request processing,

146 handling prompt preparation and engine input.

147 """

148 +

149 request_prompts: Optional[Sequence[RequestPrompt]] = []

150 engine_prompts: Optional[Union[list[EngineTokensPrompt],

151 list[EngineEmbedsPrompt]]] = []

@@ -147,6 +158,7 @@ class ResponseGenerationMixin(BaseModel):

158 Mixin for response generation,

159 managing result generators and final batch results.

160 """

161 +

162 result_generator: Optional[AsyncGenerator[tuple[int, Union[

163 RequestOutput, PoolingRequestOutput]], None]] = None

164 final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(

@@ -155,8 +167,12 @@ class ResponseGenerationMixin(BaseModel):

167 model_config = ConfigDict(arbitrary_types_allowed=True)

168

169

158 -class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, BaseModel,

159 - Generic[RequestT]):

170 +class ServeContext(

171 + RequestProcessingMixin,

172 + ResponseGenerationMixin,

173 + BaseModel,

174 + Generic[RequestT],

175 +):

176 # Shared across all requests

177 request: RequestT

178 raw_request: Optional[Request] = None

@@ -298,8 +314,8 @@ class OpenAIServing:

314 truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens",

315 None)

316

301 - if truncate_prompt_tokens is not None and \

302 - truncate_prompt_tokens > self.max_model_len:

317 + if (truncate_prompt_tokens is not None

318 + and truncate_prompt_tokens > self.max_model_len):

319 return self.create_error_response(

320 "truncate_prompt_tokens value is "

321 "greater than max_model_len."

@@ -344,10 +360,12 @@ class OpenAIServing:

360 return self.create_error_response(

361 "Request prompts not available")

362

347 - self._log_inputs(request_id_item,

348 - ctx.request_prompts[i],

349 - params=pooling_params,

350 - lora_request=ctx.lora_request)

363 + self._log_inputs(

364 + request_id_item,

365 + ctx.request_prompts[i],

366 + params=pooling_params,

367 + lora_request=ctx.lora_request,

368 + )

369

370 # Mypy has an existing bug related to inferring the variance of

371 # TypedDicts with `builtins.enumerate`:

@@ -410,10 +428,11 @@ class OpenAIServing:

428 return self.create_error_response(str(e))

429

430 def create_error_response(

413 - self,

414 - message: str,

415 - err_type: str = "BadRequestError",

416 - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:

431 + self,

432 + message: str,

433 + err_type: str = "BadRequestError",

434 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,

435 + ) -> ErrorResponse:

436 if self.log_error_stack:

437 exc_type, _, _ = sys.exc_info()

438 if exc_type is not None:

@@ -424,10 +443,11 @@ class OpenAIServing:

443 message=message, type=err_type, code=status_code.value))

444

445 def create_streaming_error_response(

427 - self,

428 - message: str,

429 - err_type: str = "BadRequestError",

430 - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str:

446 + self,

447 + message: str,

448 + err_type: str = "BadRequestError",

449 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,

450 + ) -> str:

451 json_str = json.dumps(

452 self.create_error_response(message=message,

453 err_type=err_type,

@@ -438,25 +458,25 @@ class OpenAIServing:

458 self,

459 request: AnyRequest,

460 ) -> Optional[ErrorResponse]:

441 -

461 error_response = None

462

463 if self._is_model_supported(request.model):

464 return None

465 if request.model in self.models.lora_requests:

466 return None

448 - if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING and request.model and (

449 - load_result := await self.models.resolve_lora(request.model)):

467 + if (envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING and request.model and

468 + (load_result := await self.models.resolve_lora(request.model))):

469 if isinstance(load_result, LoRARequest):

470 return None

452 - if isinstance(load_result, ErrorResponse) and \

453 - load_result.error.code == HTTPStatus.BAD_REQUEST.value:

471 + if (isinstance(load_result, ErrorResponse) and

472 + load_result.error.code == HTTPStatus.BAD_REQUEST.value):

473 error_response = load_result

474

475 return error_response or self.create_error_response(

476 message=f"The model `{request.model}` does not exist.",

477 err_type="NotFoundError",

459 - status_code=HTTPStatus.NOT_FOUND)

478 + status_code=HTTPStatus.NOT_FOUND,

479 + )

480

481 def _get_active_default_mm_loras(

482 self, request: AnyRequest) -> Optional[LoRARequest]:

@@ -487,7 +507,6 @@ class OpenAIServing:

507 request: AnyRequest,

508 supports_default_mm_loras: bool = False,

509 ) -> Optional[LoRARequest]:

490 -

510 if request.model in self.models.lora_requests:

511 return self.models.lora_requests[request.model]

512

@@ -548,13 +567,15 @@ class OpenAIServing:

567 prompt,

568 add_special_tokens=add_special_tokens,

569 truncation=True,

551 - max_length=self.max_model_len)

570 + max_length=self.max_model_len,

571 + )

572 else:

573 encoded = await async_tokenizer(

574 prompt,

575 add_special_tokens=add_special_tokens,

576 truncation=True,

557 - max_length=truncate_prompt_tokens)

577 + max_length=truncate_prompt_tokens,

578 + )

579

580 input_ids = encoded.input_ids

581 input_text = prompt

@@ -595,16 +616,22 @@ class OpenAIServing:

616

617 # Note: EmbeddingRequest, ClassificationRequest,

618 # and ScoreRequest doesn't have max_tokens

598 - if isinstance(request,

599 - (EmbeddingChatRequest, EmbeddingCompletionRequest,

600 - ScoreRequest, RerankRequest, ClassificationRequest)):

601 -

619 + if isinstance(

620 + request,

621 + (

622 + EmbeddingChatRequest,

623 + EmbeddingCompletionRequest,

624 + ScoreRequest,

625 + RerankRequest,

626 + ClassificationRequest,

627 + ),

628 + ):

629 # Note: input length can be up to the entire model context length

630 # since these requests don't generate tokens.

631 if token_num > self.max_model_len:

632 operations: dict[type[AnyRequest], str] = {

633 ScoreRequest: "score",

607 - ClassificationRequest: "classification"

634 + ClassificationRequest: "classification",

635 }

636 operation = operations.get(type(request),

637 "embedding generation")

@@ -618,8 +645,11 @@ class OpenAIServing:

645

646 # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens

647 # and does not require model context length validation

621 - if isinstance(request, (TokenizeCompletionRequest, TokenizeChatRequest,

622 - DetokenizeRequest)):

648 + if isinstance(

649 + request,

650 + (TokenizeCompletionRequest, TokenizeChatRequest,

651 + DetokenizeRequest),

652 + ):

653 return TextTokensPrompt(prompt=input_text,

654 prompt_token_ids=input_ids)

655

@@ -639,8 +669,8 @@ class OpenAIServing:

669 f"{token_num} input tokens. Please reduce the length of "

670 "the input messages.")

671

642 - if max_tokens is not None and \

643 - token_num + max_tokens > self.max_model_len:

672 + if (max_tokens is not None

673 + and token_num + max_tokens > self.max_model_len):

674 raise ValueError(

675 "'max_tokens' or 'max_completion_tokens' is too large: "

676 f"{max_tokens}. This model's maximum context length is "

@@ -745,13 +775,14 @@ class OpenAIServing:

775 tasks = []

776 for prompt_input in batch_inputs:

777 if prompt_input["is_tokens"] is False:

748 - assert tokenizer is not None, \

749 - "Tokenizer is required for text prompts"

778 + assert tokenizer is not None, (

779 + "Tokenizer is required for text prompts")

780 task = self._normalize_prompt_text_to_input(

781 request,

782 prompt_input["content"],

783 tokenizer=tokenizer,

754 - add_special_tokens=add_special_tokens)

784 + add_special_tokens=add_special_tokens,

785 + )

786 else:

787 task = self._normalize_prompt_tokens_to_input(

788 request, prompt_input["content"], tokenizer=tokenizer)

@@ -766,9 +797,14 @@ class OpenAIServing:

797 @overload

798 async def _preprocess_completion(

799 self,

769 - request: Union[DetokenizeRequest, EmbeddingCompletionRequest,

770 - RerankRequest, ClassificationRequest, ScoreRequest,

771 - TokenizeCompletionRequest],

800 + request: Union[

801 + DetokenizeRequest,

802 + EmbeddingCompletionRequest,

803 + RerankRequest,

804 + ClassificationRequest,

805 + ScoreRequest,

806 + TokenizeCompletionRequest,

807 + ],

808 tokenizer: Optional[AnyTokenizer],

809 input_or_inputs: Union[str, list[str], list[int], list[list[int]]],

810 add_special_tokens: bool = ...,

@@ -783,8 +819,10 @@ class OpenAIServing:

819 input_or_inputs: Optional[Union[str, list[str], list[int],

820 list[list[int]]]],

821 add_special_tokens: bool = ...,

786 - ) -> tuple[list[Union[TextTokensPrompt, EmbedsPrompt]], list[Union[

787 - EngineTokensPrompt, EngineEmbedsPrompt]]]:

822 + ) -> tuple[

823 + list[Union[TextTokensPrompt, EmbedsPrompt]],

824 + list[Union[EngineTokensPrompt, EngineEmbedsPrompt]],

825 + ]:

826 ...

827

828 async def _preprocess_completion(

@@ -794,32 +832,38 @@ class OpenAIServing:

832 input_or_inputs: Optional[Union[str, list[str], list[int],

833 list[list[int]]]],

834 add_special_tokens: bool = True,

797 - ) -> tuple[Union[list[TextTokensPrompt], list[Union[

798 - TextTokensPrompt, EmbedsPrompt]]], Union[

799 - list[EngineTokensPrompt], list[Union[EngineTokensPrompt,

800 - EngineEmbedsPrompt]]]]:

801 - if not isinstance(request,

802 - CompletionRequest) and input_or_inputs is None:

835 + ) -> tuple[

836 + Union[list[TextTokensPrompt], list[Union[TextTokensPrompt,

837 + EmbedsPrompt]]],

838 + Union[

839 + list[EngineTokensPrompt],

840 + list[Union[EngineTokensPrompt, EngineEmbedsPrompt]],

841 + ],

842 + ]:

843 + if (not isinstance(request, CompletionRequest)

844 + and input_or_inputs is None):

845 raise ValueError(

846 "Prompt embeds with non-completion requests is not"

847 " currently supported.")

848

807 - (request_prompts_text, request_prompts_embeds

808 - ) = await self._tokenize_prompt_input_or_inputs_async(

809 - request,

810 - tokenizer,

811 - input_or_inputs,

812 - add_special_tokens=add_special_tokens,

813 - )

849 + (

850 + request_prompts_text,

851 + request_prompts_embeds,

852 + ) = await self._tokenize_prompt_input_or_inputs_async(

853 + request,

854 + tokenizer,

855 + input_or_inputs,

856 + add_special_tokens=add_special_tokens,

857 + )

858

859 engine_prompts_text = [

860 EngineTokensPrompt(

861 prompt_token_ids=request_prompt_text["prompt_token_ids"])

862 for request_prompt_text in request_prompts_text

863 ]

820 - cache_salt = request.cache_salt if (

821 - hasattr(request, "cache_salt")

822 - and request.cache_salt is not None) else None

864 + cache_salt = (request.cache_salt if

865 + (hasattr(request, "cache_salt")

866 + and request.cache_salt is not None) else None)

867 if cache_salt:

868 for prompt_text in engine_prompts_text:

869 prompt_text["cache_salt"] = cache_salt

@@ -831,8 +875,8 @@ class OpenAIServing:

875 # non-completion requests and if we don't add the overload here,

876 # everywhere this function is used outside of serving_completion will

877 # need logic asserting that only text prompts are in the request.

834 - if not isinstance(request,

835 - CompletionRequest) and input_or_inputs is not None:

878 + if (not isinstance(request, CompletionRequest)

879 + and input_or_inputs is not None):

880 return request_prompts_text, engine_prompts_text

881

882 engine_prompts_embeds = [

@@ -862,8 +906,11 @@ class OpenAIServing:

906 chat_template_kwargs: Optional[dict[str, Any]] = None,

907 tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,

908 add_special_tokens: bool = False,

865 - ) -> tuple[list[ConversationMessage], Sequence[RequestPrompt],

866 - list[EngineTokensPrompt]]:

909 + ) -> tuple[

910 + list[ConversationMessage],

911 + Sequence[RequestPrompt],

912 + list[EngineTokensPrompt],

913 + ]:

914 model_config = self.model_config

915

916 resolved_content_format = resolve_chat_template_content_format(

@@ -925,8 +972,8 @@ class OpenAIServing:

972

973 if tokenizer is None:

974 assert isinstance(request_prompt, str), (

928 - "Prompt has to be a string", \

929 - "when the tokenizer is not initialised"

975 + "Prompt has to be a string",

976 + "when the tokenizer is not initialised",

977 )

978 prompt_inputs = TextTokensPrompt(prompt=request_prompt,

979 prompt_token_ids=[1])

@@ -943,7 +990,8 @@ class OpenAIServing:

990 "Prompt has to be either a string or a list of token ids")

991 prompt_inputs = TextTokensPrompt(

992 prompt=tokenizer.decode(request_prompt),

946 - prompt_token_ids=request_prompt)

993 + prompt_token_ids=request_prompt,

994 + )

995

996 engine_prompt = EngineTokensPrompt(

997 prompt_token_ids=prompt_inputs["prompt_token_ids"])

@@ -1007,22 +1055,23 @@ class OpenAIServing:

1055 prompt_token_ids=prompt_token_ids)

1056 request_prompt = prompt_token_ids

1057 # Update the sampling params.

1010 - sampling_params.max_tokens = (self.max_model_len -

1011 - len(prompt_token_ids))

1058 + sampling_params.max_tokens = self.max_model_len - len(

1059 + prompt_token_ids)

1060 # OPTIMIZATION

1061 priority = orig_priority - 1

1062

1063 @staticmethod

1064 def _load_prompt_embeds(

1065 prompt_embeds: Optional[Union[bytes, list[bytes]]],

1018 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

1066 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

1067 ) -> list[EmbedsPrompt]:

1068

1069 def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:

1022 - tensor = torch.load(io.BytesIO(

1023 - pybase64.b64decode(embed, validate=True)),

1024 - weights_only=True,

1025 - map_location=torch.device("cpu"))

1070 + tensor = torch.load(

1071 + io.BytesIO(pybase64.b64decode(embed, validate=True)),

1072 + weights_only=True,

1073 + map_location=torch.device("cpu"),

1074 + )

1075 assert isinstance(tensor, torch.Tensor) and tensor.dtype in (

1076 torch.float32,

1077 torch.bfloat16,

@@ -1061,7 +1110,7 @@ class OpenAIServing:

1110 prompt = inputs

1111 elif isinstance(inputs, list):

1112 prompt_token_ids = inputs

1064 - elif 'prompt_embeds' in inputs:

1113 + elif "prompt_embeds" in inputs:

1114 prompt_embeds = inputs.get("prompt_embeds")

1115 else:

1116 prompt = inputs["prompt"]

@@ -1101,10 +1150,12 @@ class OpenAIServing:

1150 return raw_request.headers.get("X-Request-Id", default)

1151

1152 @staticmethod

1104 - def _get_decoded_token(logprob: Logprob,

1105 - token_id: int,

1106 - tokenizer: AnyTokenizer,

1107 - return_as_token_id: bool = False) -> str:

1153 + def _get_decoded_token(

1154 + logprob: Logprob,

1155 + token_id: int,

1156 + tokenizer: AnyTokenizer,

1157 + return_as_token_id: bool = False,

1158 + ) -> str:

1159 if return_as_token_id:

1160 return f"token_id:{token_id}"

1161

@@ -1117,9 +1168,11 @@ class OpenAIServing:

1168 return True

1169 return self.models.is_base_model(model_name)

1170

1120 - def _get_model_name(self,

1121 - model_name: Optional[str] = None,

1122 - lora_request: Optional[LoRARequest] = None) -> str:

1171 + def _get_model_name(

1172 + self,

1173 + model_name: Optional[str] = None,

1174 + lora_request: Optional[LoRARequest] = None,

1175 + ) -> str:

1176 if lora_request:

1177 return lora_request.lora_name

1178 if not model_name:

@@ -1129,7 +1182,7 @@ class OpenAIServing:

1182

1183 def clamp_prompt_logprobs(

1184 prompt_logprobs: Union[PromptLogprobs,

1132 - None]) -> Union[PromptLogprobs, None]:

1185 + None], ) -> Union[PromptLogprobs, None]:

1186 if prompt_logprobs is None:

1187 return prompt_logprobs

1188

@@ -1137,6 +1190,6 @@ def clamp_prompt_logprobs(

1190 if logprob_dict is None:

1191 continue

1192 for logprob_values in logprob_dict.values():

1140 - if logprob_values.logprob == float('-inf'):

1193 + if logprob_values.logprob == float("-inf"):

1194 logprob_values.logprob = -9999.0

1195 return prompt_logprobs