b47840019e · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -394,18 +394,24 @@ def run_eagle2_5(questions: list[str], modality: str) -> ModelRequestData:

394 def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:

395 model_name = "baidu/ERNIE-4.5-VL-28B-A3B-PT"

396

397 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

398 engine_args = EngineArgs(

399 model=model_name,

400 max_model_len=4096,

401 max_num_seqs=5,

401 - limit_mm_per_prompt={modality: 1},

402 + limit_mm_per_prompt=mm_limit,

403 trust_remote_code=True,

404 )

405

406 + image_placeholder = "Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>"

407 + video_placeholder = "Video 1:<|VIDEO_START|><|video@placeholder|><|VIDEO_END|>"

408 +

409 if modality == "image":

406 - placeholder = "Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>"

410 + placeholder = image_placeholder

411 elif modality == "video":

408 - placeholder = "Video 1:<|VIDEO_START|><|video@placeholder|><|VIDEO_END|>"

412 + placeholder = video_placeholder

413 + elif modality == "image+video":

414 + placeholder = image_placeholder + video_placeholder

415

416 prompts = [

417 (

@@ -425,6 +431,7 @@ def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:

431 def run_exaone4_5(questions: list[str], modality: str) -> ModelRequestData:

432 model_name = "LGAI-EXAONE/EXAONE-4.5-33B"

433

434 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

435 engine_args = EngineArgs(

436 model=model_name,

437 max_model_len=4096,

@@ -434,18 +441,23 @@ def run_exaone4_5(questions: list[str], modality: str) -> ModelRequestData:

441 "max_pixels": 1280 * 28 * 28,

442 "fps": 1,

443 },

437 - limit_mm_per_prompt={modality: 1},

444 + limit_mm_per_prompt=mm_limit,

445 )

446

447 + image_placeholder = "<vision><|image_pad|></vision>"

448 + video_placeholder = "<vision><|video_pad|></vision>"

449 +

450 if modality == "image":

441 - placeholder = "<|image_pad|>"

451 + placeholder = image_placeholder

452 elif modality == "video":

443 - placeholder = "<|video_pad|>"

453 + placeholder = video_placeholder

454 + elif modality == "image+video":

455 + placeholder = image_placeholder + video_placeholder

456

457 prompts = [

458 (

459 "<|system|>\nYou are a helpful assistant.<|endofturn|>\n"

448 - f"<|user|>\n<vision>{placeholder}</vision>"

460 + f"<|user|>\n{placeholder}"

461 f"{question}<|endofturn|>\n"

462 "<|assistant|>\n"

463 )

@@ -566,6 +578,7 @@ def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:

578 def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:

579 model_name = "zai-org/GLM-4.1V-9B-Thinking"

580

581 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

582 engine_args = EngineArgs(

583 model=model_name,

584 max_model_len=4096,

@@ -574,14 +587,19 @@ def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:

587 "size": {"shortest_edge": 12544, "longest_edge": 47040000},

588 "fps": 1,

589 },

577 - limit_mm_per_prompt={modality: 1},

590 + limit_mm_per_prompt=mm_limit,

591 enforce_eager=True,

592 )

593

594 + image_placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

595 + video_placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

596 +

597 if modality == "image":

582 - placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

598 + placeholder = image_placeholder

599 elif modality == "video":

584 - placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

600 + placeholder = video_placeholder

601 + elif modality == "image+video":

602 + placeholder = image_placeholder + video_placeholder

603

604 prompts = [

605 (

@@ -602,6 +620,7 @@ def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:

620 def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:

621 model_name = "zai-org/GLM-4.5V"

622

623 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

624 engine_args = EngineArgs(

625 model=model_name,

626 max_model_len=4096,

@@ -610,15 +629,20 @@ def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:

629 "size": {"shortest_edge": 12544, "longest_edge": 47040000},

630 "fps": 1,

631 },

613 - limit_mm_per_prompt={modality: 1},

632 + limit_mm_per_prompt=mm_limit,

633 enforce_eager=True,

634 tensor_parallel_size=4,

635 )

636

637 + image_placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

638 + video_placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

639 +

640 if modality == "image":

619 - placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

641 + placeholder = image_placeholder

642 elif modality == "video":

621 - placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

643 + placeholder = video_placeholder

644 + elif modality == "image+video":

645 + placeholder = image_placeholder + video_placeholder

646

647 prompts = [

648 (

@@ -639,6 +663,7 @@ def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:

663 def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:

664 model_name = "zai-org/GLM-4.5V-FP8"

665

666 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

667 engine_args = EngineArgs(

668 model=model_name,

669 max_model_len=4096,

@@ -647,15 +672,20 @@ def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:

672 "size": {"shortest_edge": 12544, "longest_edge": 47040000},

673 "fps": 1,

674 },

650 - limit_mm_per_prompt={modality: 1},

675 + limit_mm_per_prompt=mm_limit,

676 enforce_eager=True,

677 tensor_parallel_size=4,

678 )

679

680 + image_placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

681 + video_placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

682 +

683 if modality == "image":

656 - placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

684 + placeholder = image_placeholder

685 elif modality == "video":

658 - placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

686 + placeholder = video_placeholder

687 + elif modality == "image+video":

688 + placeholder = image_placeholder + video_placeholder

689

690 prompts = [

691 (

@@ -676,6 +706,7 @@ def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:

706 def run_glm_ocr(questions: list[str], modality: str) -> ModelRequestData:

707 model_name = "zai-org/GLM-OCR"

708

709 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

710 engine_args = EngineArgs(

711 model=model_name,

712 max_model_len=4096,

@@ -684,14 +715,19 @@ def run_glm_ocr(questions: list[str], modality: str) -> ModelRequestData:

715 "size": {"shortest_edge": 12544, "longest_edge": 47040000},

716 "fps": 1,

717 },

687 - limit_mm_per_prompt={modality: 1},

718 + limit_mm_per_prompt=mm_limit,

719 enforce_eager=True,

720 )

721

722 + image_placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

723 + video_placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

724 +

725 if modality == "image":

692 - placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

726 + placeholder = image_placeholder

727 elif modality == "video":

694 - placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

728 + placeholder = video_placeholder

729 + elif modality == "image+video":

730 + placeholder = image_placeholder + video_placeholder

731

732 prompts = [

733 (

@@ -772,11 +808,12 @@ def run_hyperclovax_seed_vision(

808 model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"

809 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

810

811 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

812 engine_args = EngineArgs(

813 model=model_name,

814 trust_remote_code=True,

778 - max_model_len=8192 if modality == "image" else 16384,

779 - limit_mm_per_prompt={modality: 1},

815 + max_model_len=16384 if modality in ("video", "image+video") else 8192,

816 + limit_mm_per_prompt=mm_limit,

817 )

818

819 messages = list()

@@ -828,6 +865,29 @@ def run_hyperclovax_seed_vision(

865 }

866 ]

867 )

868 + elif modality == "image+video":

869 + messages.append(

870 + [

871 + {

872 + "role": "user",

873 + "content": [

874 + {

875 + "type": "image",

876 + "ocr": "",

877 + "lens_keywords": "",

878 + "lens_local_keywords": "",

879 + },

880 + {

881 + "type": "video",

882 + },

883 + {

884 + "type": "text",

885 + "text": question,

886 + },

887 + ],

888 + }

889 + ]

890 + )

891 else:

892 raise ValueError(f"Unsupported modality: {modality}")

893

@@ -876,19 +936,25 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:

936 def run_interns1(questions: list[str], modality: str) -> ModelRequestData:

937 model_name = "internlm/Intern-S1-mini"

938

939 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

940 engine_args = EngineArgs(

941 model=model_name,

942 trust_remote_code=True,

943 max_model_len=8192,

944 max_num_seqs=2,

884 - limit_mm_per_prompt={modality: 1},

945 + limit_mm_per_prompt=mm_limit,

946 enforce_eager=True,

947 )

948

949 + image_placeholder = "<IMG_CONTEXT>"

950 + video_placeholder = "<video>"

951 +

952 if modality == "image":

889 - placeholder = "<IMG_CONTEXT>"

953 + placeholder = image_placeholder

954 elif modality == "video":

891 - placeholder = "<video>"

955 + placeholder = video_placeholder

956 + elif modality == "image+video":

957 + placeholder = image_placeholder + "\n" + video_placeholder

958

959 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

960 messages = [

@@ -909,20 +975,26 @@ def run_interns1(questions: list[str], modality: str) -> ModelRequestData:

975 def run_interns1_pro(questions: list[str], modality: str) -> ModelRequestData:

976 model_name = "internlm/Intern-S1-Pro"

977

978 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

979 engine_args = EngineArgs(

980 model=model_name,

981 trust_remote_code=True,

982 max_model_len=8192,

983 max_num_seqs=2,

917 - limit_mm_per_prompt={modality: 1},

984 + limit_mm_per_prompt=mm_limit,

985 enforce_eager=True,

986 tensor_parallel_size=4,

987 )

988

989 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

990 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

991 +

992 if modality == "image":

923 - placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

993 + placeholder = image_placeholder

994 elif modality == "video":

925 - placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

995 + placeholder = video_placeholder

996 + elif modality == "image+video":

997 + placeholder = image_placeholder + video_placeholder

998

999 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

1000 messages = [

@@ -943,17 +1015,23 @@ def run_interns1_pro(questions: list[str], modality: str) -> ModelRequestData:

1015 def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

1016 model_name = "OpenGVLab/InternVL3-2B"

1017

1018 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1019 engine_args = EngineArgs(

1020 model=model_name,

1021 trust_remote_code=True,

1022 max_model_len=8192,

950 - limit_mm_per_prompt={modality: 1},

1023 + limit_mm_per_prompt=mm_limit,

1024 )

1025

1026 + image_placeholder = "<image>"

1027 + video_placeholder = "<video>"

1028 +

1029 if modality == "image":

954 - placeholder = "<image>"

1030 + placeholder = image_placeholder

1031 elif modality == "video":

956 - placeholder = "<video>"

1032 + placeholder = video_placeholder

1033 + elif modality == "image+video":

1034 + placeholder = image_placeholder + "\n" + video_placeholder

1035

1036 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

1037 messages = [

@@ -1010,21 +1088,27 @@ def run_kanana_v(questions: list[str], modality: str) -> ModelRequestData:

1088 def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:

1089 model_name = "Kwai-Keye/Keye-VL-8B-Preview"

1090

1091 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1092 engine_args = EngineArgs(

1093 model=model_name,

1094 max_model_len=8192,

1095 trust_remote_code=True,

1017 - limit_mm_per_prompt={modality: 1},

1096 + limit_mm_per_prompt=mm_limit,

1097 )

1098

1099 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

1100 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

1101 +

1102 if modality == "image":

1021 - placeholder = "<|image_pad|>"

1103 + placeholder = image_placeholder

1104 elif modality == "video":

1023 - placeholder = "<|video_pad|>"

1105 + placeholder = video_placeholder

1106 + elif modality == "image+video":

1107 + placeholder = image_placeholder + video_placeholder

1108

1109 prompts = [

1110 (

1027 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

1111 + f"<|im_start|>user\n{placeholder}"

1112 f"{question}<|im_end|>\n"

1113 "<|im_start|>assistant\n"

1114 )

@@ -1041,21 +1125,27 @@ def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:

1125 def run_keye_vl1_5(questions: list[str], modality: str) -> ModelRequestData:

1126 model_name = "Kwai-Keye/Keye-VL-1.5-8B"

1127

1128 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1129 engine_args = EngineArgs(

1130 model=model_name,

1131 max_model_len=8192,

1132 trust_remote_code=True,

1048 - limit_mm_per_prompt={modality: 1},

1133 + limit_mm_per_prompt=mm_limit,

1134 )

1135

1136 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

1137 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

1138 +

1139 if modality == "image":

1052 - placeholder = "<|image_pad|>"

1140 + placeholder = image_placeholder

1141 elif modality == "video":

1054 - placeholder = "<|video_pad|>"

1142 + placeholder = video_placeholder

1143 + elif modality == "image+video":

1144 + placeholder = image_placeholder + video_placeholder

1145

1146 prompts = [

1147 (

1058 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

1148 + f"<|im_start|>user\n{placeholder}"

1149 f"{question}<|im_end|>\n"

1150 "<|im_start|>assistant\n"

1151 )

@@ -1259,22 +1349,26 @@ def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestDat

1349

1350 # LLaVA-OneVision

1351 def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:

1262 - if modality == "video":

1263 - prompts = [

1264 - f"<|im_start|>user <video>\n{question}<|im_end|><|im_start|>assistant\n"

1265 - for question in questions

1266 - ]

1352 + image_placeholder = "<image>"

1353 + video_placeholder = "<video>"

1354

1268 - elif modality == "image":

1269 - prompts = [

1270 - f"<|im_start|>user <image>\n{question}<|im_end|><|im_start|>assistant\n"

1271 - for question in questions

1272 - ]

1355 + if modality == "image":

1356 + placeholder = image_placeholder

1357 + elif modality == "video":

1358 + placeholder = video_placeholder

1359 + elif modality == "image+video":

1360 + placeholder = image_placeholder + "\n" + video_placeholder

1361 +

1362 + prompts = [

1363 + (f"<|im_start|>user {placeholder}\n{question}<|im_end|><|im_start|>assistant\n")

1364 + for question in questions

1365 + ]

1366

1367 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1368 engine_args = EngineArgs(

1369 model="llava-hf/llava-onevision-qwen2-7b-ov-hf",

1370 max_model_len=16384,

1277 - limit_mm_per_prompt={modality: 1},

1371 + limit_mm_per_prompt=mm_limit,

1372 )

1373

1374 return ModelRequestData(

@@ -1307,7 +1401,7 @@ def run_mantis(questions: list[str], modality: str) -> ModelRequestData:

1401

1402 # MiniCPM-V

1403 def run_minicpmv_base(questions: list[str], modality: str, model_name):

1310 - assert modality in ["image", "video"]

1404 + assert modality in ["image", "video", "image+video"]

1405 # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa

1406

1407 # 2.0

@@ -1329,12 +1423,13 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):

1423 # o2.6: image, video, audio

1424 # model_name = "openbmb/MiniCPM-o-2_6"

1425 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

1426 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1427 engine_args = EngineArgs(

1428 model=model_name,

1429 max_model_len=4096,

1430 max_num_seqs=2,

1431 trust_remote_code=True,

1337 - limit_mm_per_prompt={modality: 1},

1432 + limit_mm_per_prompt=mm_limit,

1433 )

1434 # NOTE The stop_token_ids are different for various versions of MiniCPM-V

1435 # 2.0

@@ -1347,17 +1442,22 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):

1442 stop_tokens = ["<|im_end|>", "<|endoftext|>"]

1443 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]

1444

1350 - modality_placeholder = {

1351 - "image": "(<image>./</image>)",

1352 - "video": "(<video>./</video>)",

1353 - }

1445 + image_placeholder = "(<image>./</image>)"

1446 + video_placeholder = "(<video>./</video>)"

1447 +

1448 + if modality == "image":

1449 + placeholder = image_placeholder

1450 + elif modality == "video":

1451 + placeholder = video_placeholder

1452 + elif modality == "image+video":

1453 + placeholder = image_placeholder + "\n" + video_placeholder

1454

1455 prompts = [

1456 tokenizer.apply_chat_template(

1457 [

1458 {

1459 "role": "user",

1360 - "content": f"{modality_placeholder[modality]}\n{question}",

1460 + "content": f"{placeholder}\n{question}",

1461 }

1462 ],

1463 tokenize=False,

@@ -1466,20 +1566,24 @@ def run_molmo(questions: list[str], modality: str) -> ModelRequestData:

1566 def run_molmo2(questions: list[str], modality: str) -> ModelRequestData:

1567 model_name = "allenai/Molmo2-8B"

1568

1569 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1570 engine_args = EngineArgs(

1571 model=model_name,

1572 trust_remote_code=True,

1573 dtype="bfloat16",

1473 - limit_mm_per_prompt={modality: 1},

1574 + limit_mm_per_prompt=mm_limit,

1575 max_num_batched_tokens=36864,

1576 )

1577

1578 + image_placeholder = "<|image|>"

1579 + video_placeholder = "<|video|>"

1580 +

1581 if modality == "image":

1478 - placeholder = "<|image|>"

1582 + placeholder = image_placeholder

1583 elif modality == "video":

1480 - placeholder = "<|video|>"

1481 - else:

1482 - raise ValueError(f"Unsupported modality for molmo2: {modality}")

1584 + placeholder = video_placeholder

1585 + elif modality == "image+video":

1586 + placeholder = image_placeholder + video_placeholder

1587

1588 prompts = [

1589 f"{placeholder}<|im_start|>user\n{question}<|im_end|>\n<|im_start|>assistant\n"

@@ -1563,19 +1667,25 @@ def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:

1667 def run_openpangu_vl(questions: list[str], modality: str) -> ModelRequestData:

1668 model_name = "FreedomIntelligence/openPangu-VL-7B"

1669

1670 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1671 engine_args = EngineArgs(

1672 model=model_name,

1673 max_model_len=4096,

1674 max_num_seqs=4,

1675 trust_remote_code=True,

1676 enforce_eager=True,

1572 - limit_mm_per_prompt={modality: 1},

1677 + limit_mm_per_prompt=mm_limit,

1678 )

1679

1680 + image_placeholder = "[unused19]"

1681 + video_placeholder = "[unused32]"

1682 +

1683 if modality == "image":

1576 - placeholder = "[unused19]"

1684 + placeholder = image_placeholder

1685 elif modality == "video":

1578 - placeholder = "[unused32]"

1686 + placeholder = video_placeholder

1687 + elif modality == "image+video":

1688 + placeholder = image_placeholder + video_placeholder

1689

1690 prompts = [

1691 (

@@ -1623,18 +1733,25 @@ def run_ovis(questions: list[str], modality: str) -> ModelRequestData:

1733 def run_ovis2_5(questions: list[str], modality: str) -> ModelRequestData:

1734 model_name = "AIDC-AI/Ovis2.5-2B"

1735

1736 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1737 engine_args = EngineArgs(

1738 model=model_name,

1739 max_model_len=4096,

1740 max_num_seqs=2,

1741 trust_remote_code=True,

1742 dtype="half",

1632 - limit_mm_per_prompt={modality: 1},

1743 + limit_mm_per_prompt=mm_limit,

1744 )

1745 +

1746 + image_placeholder = "<image>"

1747 + video_placeholder = "<video>"

1748 +

1749 if modality == "image":

1635 - placeholder = "<image>"

1750 + placeholder = image_placeholder

1751 elif modality == "video":

1637 - placeholder = "<video>"

1752 + placeholder = video_placeholder

1753 + elif modality == "image+video":

1754 + placeholder = image_placeholder + "\n" + video_placeholder

1755

1756 prompts = [

1757 f"<|im_start|>user\n\n{placeholder}\n{question}<|im_end|>\n<|im_start|>assistant\n"

@@ -1846,6 +1963,7 @@ def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:

1963 def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:

1964 model_name = "Qwen/Qwen2-VL-7B-Instruct"

1965

1966 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

1967 engine_args = EngineArgs(

1968 model=model_name,

1969 max_model_len=4096,

@@ -1855,18 +1973,23 @@ def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:

1973 "min_pixels": 28 * 28,

1974 "max_pixels": 1280 * 28 * 28,

1975 },

1858 - limit_mm_per_prompt={modality: 1},

1976 + limit_mm_per_prompt=mm_limit,

1977 )

1978

1979 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

1980 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

1981 +

1982 if modality == "image":

1862 - placeholder = "<|image_pad|>"

1983 + placeholder = image_placeholder

1984 elif modality == "video":

1864 - placeholder = "<|video_pad|>"

1985 + placeholder = video_placeholder

1986 + elif modality == "image+video":

1987 + placeholder = image_placeholder + video_placeholder

1988

1989 prompts = [

1990 (

1991 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1869 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

1992 + f"<|im_start|>user\n{placeholder}"

1993 f"{question}<|im_end|>\n"

1994 "<|im_start|>assistant\n"

1995 )

@@ -1883,6 +2006,7 @@ def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:

2006 def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:

2007 model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

2008

2009 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2010 engine_args = EngineArgs(

2011 model=model_name,

2012 max_model_len=4096,

@@ -1892,18 +2016,23 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:

2016 "max_pixels": 1280 * 28 * 28,

2017 "fps": 1,

2018 },

1895 - limit_mm_per_prompt={modality: 1},

2019 + limit_mm_per_prompt=mm_limit,

2020 )

2021

2022 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

2023 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

2024 +

2025 if modality == "image":

1899 - placeholder = "<|image_pad|>"

2026 + placeholder = image_placeholder

2027 elif modality == "video":

1901 - placeholder = "<|video_pad|>"

2028 + placeholder = video_placeholder

2029 + elif modality == "image+video":

2030 + placeholder = image_placeholder + video_placeholder

2031

2032 prompts = [

2033 (

2034 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1906 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

2035 + f"<|im_start|>user\n{placeholder}"

2036 f"{question}<|im_end|>\n"

2037 "<|im_start|>assistant\n"

2038 )

@@ -1920,6 +2049,7 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:

2049 def run_qwen2_5_omni(questions: list[str], modality: str):

2050 model_name = "Qwen/Qwen2.5-Omni-7B"

2051

2052 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2053 engine_args = EngineArgs(

2054 model=model_name,

2055 max_model_len=4096,

@@ -1929,13 +2059,18 @@ def run_qwen2_5_omni(questions: list[str], modality: str):

2059 "max_pixels": 1280 * 28 * 28,

2060 "fps": 1,

2061 },

1932 - limit_mm_per_prompt={modality: 1},

2062 + limit_mm_per_prompt=mm_limit,

2063 )

2064

2065 + image_placeholder = "<|vision_bos|><|IMAGE|><|vision_eos|>"

2066 + video_placeholder = "<|vision_bos|><|VIDEO|><|vision_eos|>"

2067 +

2068 if modality == "image":

1936 - placeholder = "<|IMAGE|>"

2069 + placeholder = image_placeholder

2070 elif modality == "video":

1938 - placeholder = "<|VIDEO|>"

2071 + placeholder = video_placeholder

2072 + elif modality == "image+video":

2073 + placeholder = image_placeholder + video_placeholder

2074

2075 default_system = (

2076 "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "

@@ -1946,7 +2081,7 @@ def run_qwen2_5_omni(questions: list[str], modality: str):

2081 prompts = [

2082 (

2083 f"<|im_start|>system\n{default_system}<|im_end|>\n"

1949 - f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"

2084 + f"<|im_start|>user\n{placeholder}"

2085 f"{question}<|im_end|>\n"

2086 "<|im_start|>assistant\n"

2087 )

@@ -1962,6 +2097,7 @@ def run_qwen2_5_omni(questions: list[str], modality: str):

2097 def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:

2098 model_name = "Qwen/Qwen3-VL-4B-Instruct"

2099

2100 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2101 engine_args = EngineArgs(

2102 model=model_name,

2103 max_model_len=4096,

@@ -1971,18 +2107,23 @@ def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:

2107 "max_pixels": 1280 * 28 * 28,

2108 "fps": 1,

2109 },

1974 - limit_mm_per_prompt={modality: 1},

2110 + limit_mm_per_prompt=mm_limit,

2111 )

2112

2113 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

2114 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

2115 +

2116 if modality == "image":

1978 - placeholder = "<|image_pad|>"

2117 + placeholder = image_placeholder

2118 elif modality == "video":

1980 - placeholder = "<|video_pad|>"

2119 + placeholder = video_placeholder

2120 + elif modality == "image+video":

2121 + placeholder = image_placeholder + video_placeholder

2122

2123 prompts = [

2124 (

2125 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1985 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

2126 + f"<|im_start|>user\n{placeholder}"

2127 f"{question}<|im_end|>\n"

2128 "<|im_start|>assistant\n"

2129 )

@@ -1999,6 +2140,7 @@ def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:

2140 def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:

2141 model_name = "Qwen/Qwen3-VL-30B-A3B-Instruct"

2142

2143 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2144 engine_args = EngineArgs(

2145 model=model_name,

2146 max_model_len=4096,

@@ -2008,18 +2150,23 @@ def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:

2150 "max_pixels": 1280 * 28 * 28,

2151 "fps": 1,

2152 },

2011 - limit_mm_per_prompt={modality: 1},

2153 + limit_mm_per_prompt=mm_limit,

2154 )

2155

2156 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

2157 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

2158 +

2159 if modality == "image":

2015 - placeholder = "<|image_pad|>"

2160 + placeholder = image_placeholder

2161 elif modality == "video":

2017 - placeholder = "<|video_pad|>"

2162 + placeholder = video_placeholder

2163 + elif modality == "image+video":

2164 + placeholder = image_placeholder + video_placeholder

2165

2166 prompts = [

2167 (

2168 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

2022 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

2169 + f"<|im_start|>user\n{placeholder}"

2170 f"{question}<|im_end|>\n"

2171 "<|im_start|>assistant\n"

2172 )

@@ -2190,6 +2337,7 @@ def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:

2337 def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:

2338 model_name = "omni-research/Tarsier2-Recap-7b"

2339

2340 + mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2341 engine_args = EngineArgs(

2342 model=model_name,

2343 max_model_len=4096,

@@ -2197,18 +2345,23 @@ def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:

2345 "architectures": ["Tarsier2ForConditionalGeneration"],

2346 "model_type": "tarsier2",

2347 },

2200 - limit_mm_per_prompt={modality: 1},

2348 + limit_mm_per_prompt=mm_limit,

2349 )

2350

2351 + image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

2352 + video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

2353 +

2354 if modality == "image":

2204 - placeholder = "<|image_pad|>"

2355 + placeholder = image_placeholder

2356 elif modality == "video":

2206 - placeholder = "<|video_pad|>"

2357 + placeholder = video_placeholder

2358 + elif modality == "image+video":

2359 + placeholder = image_placeholder + video_placeholder

2360

2361 prompts = [

2362 (

2363 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

2211 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

2364 + f"<|im_start|>user\n{placeholder}"

2365 f"{question}<|im_end|>\n"

2366 "<|im_start|>assistant\n"

2367 )

@@ -2357,6 +2510,24 @@ def get_multi_modal_input(args):

2510 "questions": vision_chunk_questions,

2511 }

2512

2513 + if args.modality == "image+video":

2514 + image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")

2515 + needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA

2516 + video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays

2517 + metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata

2518 + img_video_questions = [

2519 + "What is shown in the image? What happens in the video?",

2520 + "Describe both the image and the video content.",

2521 + ]

2522 +

2523 + return {

2524 + "data": {

2525 + "image": image,

2526 + "video": ([(video, metadata)] if needs_metadata else video),

2527 + },

2528 + "questions": img_video_questions,

2529 + }

2530 +

2531 msg = f"Modality {args.modality} is not supported."

2532 raise ValueError(msg)

2533

@@ -2439,7 +2610,7 @@ def parse_args():

2610 "--modality",

2611 type=str,

2612 default="image",

2442 - choices=["image", "video", "vision_chunk"],

2613 + choices=["image", "video", "image+video", "vision_chunk"],

2614 help="Modality of the input.",

2615 )

2616 parser.add_argument(

@@ -2546,23 +2717,42 @@ def main(args):

2717 else req_data.sampling_params

2718 )

2719

2720 + def _mm_data(data, modality):

2721 + if modality == "image+video":

2722 + return {"image": data["image"], "video": data["video"]}

2723 + return {modality: data}

2724 +

2725 + def _mm_uuid(uuid, modality):

2726 + if modality == "image+video":

2727 + return {"image": uuid, "video": uuid + "v"}

2728 + return {modality: uuid}

2729 +

2730 + def _mm_empty(modality):

2731 + if modality == "image+video":

2732 + return {"image": None, "video": None}

2733 + return {modality: None}

2734 +

2735 assert args.num_prompts > 0

2736 if args.num_prompts == 1:

2737 # Single inference

2738 uuid = "uuid_0"

2739 inputs = {

2740 "prompt": prompts[0],

2555 - "multi_modal_data": {modality: data},

2556 - "multi_modal_uuids": {modality: uuid},

2741 + "multi_modal_data": _mm_data(data, modality),

2742 + "multi_modal_uuids": _mm_uuid(uuid, modality),

2743 }

2744 inputs_with_empty_media = {

2745 "prompt": prompts[0],

2560 - "multi_modal_data": {modality: None},

2561 - "multi_modal_uuids": {modality: uuid},

2746 + "multi_modal_data": _mm_empty(modality),

2747 + "multi_modal_uuids": _mm_uuid(uuid, modality),

2748 }

2749 else:

2750 # Batch inference

2751 if args.image_repeat_prob is not None:

2752 + if modality == "image+video":

2753 + raise ValueError(

2754 + "--image-repeat-prob is not supported for 'image+video' modality"

2755 + )

2756 # Repeat images with specified probability of "image_repeat_prob"

2757 inputs, inputs_with_empty_media = apply_image_repeat(

2758 args.image_repeat_prob,

@@ -2572,7 +2762,7 @@ def main(args):

2762 modality,

2763 )

2764 else:

2575 - # Use the same image for all prompts

2765 + # Use the same image/video for all prompts

2766 inputs = []

2767 inputs_with_empty_media = []

2768 for i in range(args.num_prompts):

@@ -2580,15 +2770,15 @@ def main(args):

2770 inputs.append(

2771 {

2772 "prompt": prompts[i % len(prompts)],

2583 - "multi_modal_data": {modality: data},

2584 - "multi_modal_uuids": {modality: uuid},

2773 + "multi_modal_data": _mm_data(data, modality),

2774 + "multi_modal_uuids": _mm_uuid(uuid, modality),

2775 }

2776 )

2777 inputs_with_empty_media.append(

2778 {

2779 "prompt": prompts[i % len(prompts)],

2590 - "multi_modal_data": {modality: None},

2591 - "multi_modal_uuids": {modality: uuid},

2780 + "multi_modal_data": _mm_empty(modality),

2781 + "multi_modal_uuids": _mm_uuid(uuid, modality),

2782 }

2783 )

2784

[MM][Misc] Support image+video mixed inputs (per prompt) for VLM examples (#40335)