5b8c390747 · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -45,7 +45,7 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:

45 max_model_len=4096,

46 max_num_seqs=2,

47 dtype="bfloat16",

48 - limit_mm_per_prompt={"image": 1},

48 + limit_mm_per_prompt={modality: 1},

49 )

50

51 prompts = [(f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"

@@ -71,7 +71,7 @@ def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:

71 max_model_len=2048,

72 max_num_seqs=2,

73 mm_processor_kwargs={"crop_to_patches": True},

74 - limit_mm_per_prompt={"image": 1},

74 + limit_mm_per_prompt={modality: 1},

75 )

76 prompts = [

77 f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"

@@ -92,7 +92,7 @@ def run_blip2(questions: list[str], modality: str) -> ModelRequestData:

92 prompts = [f"Question: {question} Answer:" for question in questions]

93 engine_args = EngineArgs(

94 model="Salesforce/blip2-opt-6.7b",

95 - limit_mm_per_prompt={"image": 1},

95 + limit_mm_per_prompt={modality: 1},

96 )

97

98 return ModelRequestData(

@@ -110,7 +110,7 @@ def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:

110 model="facebook/chameleon-7b",

111 max_model_len=4096,

112 max_num_seqs=2,

113 - limit_mm_per_prompt={"image": 1},

113 + limit_mm_per_prompt={modality: 1},

114 )

115

116 return ModelRequestData(

@@ -130,7 +130,7 @@ def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:

130 max_model_len=4096,

131 max_num_seqs=2,

132 hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},

133 - limit_mm_per_prompt={"image": 1},

133 + limit_mm_per_prompt={modality: 1},

134 )

135

136 prompts = [

@@ -155,7 +155,7 @@ def run_florence2(questions: list[str], modality: str) -> ModelRequestData:

155 max_num_seqs=2,

156 trust_remote_code=True,

157 dtype="bfloat16",

158 - limit_mm_per_prompt={"image": 1},

158 + limit_mm_per_prompt={modality: 1},

159 )

160

161 prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]

@@ -175,7 +175,7 @@ def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:

175 model="adept/fuyu-8b",

176 max_model_len=2048,

177 max_num_seqs=2,

178 - limit_mm_per_prompt={"image": 1},

178 + limit_mm_per_prompt={modality: 1},

179 )

180

181 return ModelRequestData(

@@ -194,7 +194,7 @@ def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:

194 max_model_len=2048,

195 max_num_seqs=2,

196 mm_processor_kwargs={"do_pan_and_scan": True},

197 - limit_mm_per_prompt={"image": 1},

197 + limit_mm_per_prompt={modality: 1},

198 )

199

200 prompts = [("<bos><start_of_turn>user\n"

@@ -219,7 +219,7 @@ def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:

219 trust_remote_code=True,

220 enforce_eager=True,

221 hf_overrides={"architectures": ["GLM4VForCausalLM"]},

222 - limit_mm_per_prompt={"image": 1},

222 + limit_mm_per_prompt={modality: 1},

223 )

224

225 prompts = [

@@ -246,7 +246,7 @@ def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:

246 model=model_name,

247 trust_remote_code=True,

248 max_model_len=8192,

249 - limit_mm_per_prompt={"image": 1},

249 + limit_mm_per_prompt={modality: 1},

250 )

251

252 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -287,7 +287,7 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:

287 "longest_edge": 3 * 364

288 },

289 },

290 - limit_mm_per_prompt={"image": 1},

290 + limit_mm_per_prompt={modality: 1},

291 )

292 prompts = [(

293 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"

@@ -314,7 +314,7 @@ def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:

314 "longest_edge": 384

315 },

316 },

317 - limit_mm_per_prompt={"image": 1},

317 + limit_mm_per_prompt={modality: 1},

318 )

319 prompts = [

320 (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")

@@ -337,7 +337,7 @@ def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

337 model=model_name,

338 trust_remote_code=True,

339 max_model_len=4096,

340 - limit_mm_per_prompt={"image": 1},

340 + limit_mm_per_prompt={modality: 1},

341 )

342

343 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -378,7 +378,7 @@ def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:

378 model="moonshotai/Kimi-VL-A3B-Instruct",

379 trust_remote_code=True,

380 max_model_len=4096,

381 - limit_mm_per_prompt={"image": 1},

381 + limit_mm_per_prompt={modality: 1},

382 )

383

384 return ModelRequestData(

@@ -398,7 +398,7 @@ def run_llava(questions: list[str], modality: str) -> ModelRequestData:

398 engine_args = EngineArgs(

399 model="llava-hf/llava-1.5-7b-hf",

400 max_model_len=4096,

401 - limit_mm_per_prompt={"image": 1},

401 + limit_mm_per_prompt={modality: 1},

402 )

403

404 return ModelRequestData(

@@ -415,7 +415,7 @@ def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:

415 engine_args = EngineArgs(

416 model="llava-hf/llava-v1.6-mistral-7b-hf",

417 max_model_len=8192,

418 - limit_mm_per_prompt={"image": 1},

418 + limit_mm_per_prompt={modality: 1},

419 )

420

421 return ModelRequestData(

@@ -437,7 +437,7 @@ def run_llava_next_video(questions: list[str],

437 model="llava-hf/LLaVA-NeXT-Video-7B-hf",

438 max_model_len=8192,

439 max_num_seqs=2,

440 - limit_mm_per_prompt={"image": 1},

440 + limit_mm_per_prompt={modality: 1},

441 )

442

443 return ModelRequestData(

@@ -465,7 +465,7 @@ def run_llava_onevision(questions: list[str],

465 engine_args = EngineArgs(

466 model="llava-hf/llava-onevision-qwen2-7b-ov-hf",

467 max_model_len=16384,

468 - limit_mm_per_prompt={"image": 1},

468 + limit_mm_per_prompt={modality: 1},

469 )

470

471 return ModelRequestData(

@@ -488,7 +488,7 @@ def run_mantis(questions: list[str], modality: str) -> ModelRequestData:

488 model="TIGER-Lab/Mantis-8B-siglip-llama3",

489 max_model_len=4096,

490 hf_overrides={"architectures": ["MantisForConditionalGeneration"]},

491 - limit_mm_per_prompt={"image": 1},

491 + limit_mm_per_prompt={modality: 1},

492 )

493 stop_token_ids = [128009]

494

@@ -529,7 +529,7 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):

529 max_model_len=4096,

530 max_num_seqs=2,

531 trust_remote_code=True,

532 - limit_mm_per_prompt={"image": 1},

532 + limit_mm_per_prompt={modality: 1},

533 )

534 # NOTE The stop_token_ids are different for various versions of MiniCPM-V

535 # 2.0

@@ -584,7 +584,7 @@ def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:

584 max_model_len=8192,

585 max_num_seqs=2,

586 tensor_parallel_size=2,

587 - limit_mm_per_prompt={"image": 1},

587 + limit_mm_per_prompt={modality: 1},

588 )

589

590 prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

@@ -610,7 +610,7 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:

610 model=model_name,

611 max_model_len=8192,

612 max_num_seqs=2,

613 - limit_mm_per_prompt={"image": 1},

613 + limit_mm_per_prompt={modality: 1},

614 )

615

616 tokenizer = AutoTokenizer.from_pretrained(model_name)

@@ -645,7 +645,7 @@ def run_llama4(questions: list[str], modality: str) -> ModelRequestData:

645 max_num_seqs=4,

646 tensor_parallel_size=8,

647 gpu_memory_utilization=0.4,

648 - limit_mm_per_prompt={"image": 1},

648 + limit_mm_per_prompt={modality: 1},

649 )

650

651 tokenizer = AutoTokenizer.from_pretrained(model_name)

@@ -680,7 +680,7 @@ def run_molmo(questions: list[str], modality: str) -> ModelRequestData:

680 model=model_name,

681 trust_remote_code=True,

682 dtype="bfloat16",

683 - limit_mm_per_prompt={"image": 1},

683 + limit_mm_per_prompt={modality: 1},

684 )

685

686 prompts = [

@@ -706,7 +706,7 @@ def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:

706 trust_remote_code=True,

707 max_model_len=4096,

708 tensor_parallel_size=4,

709 - limit_mm_per_prompt={"image": 1},

709 + limit_mm_per_prompt={modality: 1},

710 )

711

712 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -738,7 +738,7 @@ def run_ovis2(questions: list[str], modality: str) -> ModelRequestData:

738 trust_remote_code=True,

739 dtype="half",

740 hf_overrides={"architectures": ["Ovis2ForConditionalGeneration"]},

741 - limit_mm_per_prompt={"image": 1},

741 + limit_mm_per_prompt={modality: 1},

742 )

743

744 placeholder = "<image>\n"

@@ -761,7 +761,7 @@ def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:

761 prompts = ["caption en" for _ in questions]

762 engine_args = EngineArgs(

763 model="google/paligemma-3b-mix-224",

764 - limit_mm_per_prompt={"image": 1},

764 + limit_mm_per_prompt={modality: 1},

765 )

766

767 return ModelRequestData(

@@ -778,7 +778,7 @@ def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:

778 prompts = ["caption en" for _ in questions]

779 engine_args = EngineArgs(

780 model="google/paligemma2-3b-ft-docci-448",

781 - limit_mm_per_prompt={"image": 1},

781 + limit_mm_per_prompt={modality: 1},

782 )

783

784 return ModelRequestData(

@@ -815,7 +815,7 @@ def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:

815 max_num_seqs=2,

816 # Note - mm_processor_kwargs can also be passed to generate/chat calls

817 mm_processor_kwargs={"num_crops": 16},

818 - limit_mm_per_prompt={"image": 1},

818 + limit_mm_per_prompt={modality: 1},

819 )

820

821 return ModelRequestData(

@@ -849,7 +849,7 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:

849 max_lora_rank=320,

850 # Note - mm_processor_kwargs can also be passed to generate/chat calls

851 mm_processor_kwargs={"dynamic_hd": 16},

852 - limit_mm_per_prompt={"image": 1},

852 + limit_mm_per_prompt={modality: 1},

853 )

854

855 return ModelRequestData(

@@ -870,7 +870,7 @@ def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:

870 model=model_name,

871 max_model_len=6144,

872 max_num_seqs=2,

873 - limit_mm_per_prompt={"image": 1},

873 + limit_mm_per_prompt={modality: 1},

874 )

875

876 prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

@@ -891,7 +891,7 @@ def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:

891 max_model_len=1024,

892 max_num_seqs=2,

893 hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},

894 - limit_mm_per_prompt={"image": 1},

894 + limit_mm_per_prompt={modality: 1},

895 )

896

897 prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]

@@ -916,7 +916,7 @@ def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:

916 "min_pixels": 28 * 28,

917 "max_pixels": 1280 * 28 * 28,

918 },

919 - limit_mm_per_prompt={"image": 1},

919 + limit_mm_per_prompt={modality: 1},

920 )

921

922 if modality == "image":

@@ -951,7 +951,7 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:

951 "max_pixels": 1280 * 28 * 28,

952 "fps": 1,

953 },

954 - limit_mm_per_prompt={"image": 1},

954 + limit_mm_per_prompt={modality: 1},

955 )

956

957 if modality == "image":

@@ -985,7 +985,7 @@ def run_qwen2_5_omni(questions: list[str], modality: str):

985 "max_pixels": 1280 * 28 * 28,

986 "fps": [1],

987 },

988 - limit_mm_per_prompt={"image": 1},

988 + limit_mm_per_prompt={modality: 1},

989 )

990

991 if modality == "image":

@@ -1018,7 +1018,7 @@ def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:

1018 model=model_name,

1019 trust_remote_code=True,

1020 max_model_len=4096,

1021 - limit_mm_per_prompt={"image": 1},

1021 + limit_mm_per_prompt={modality: 1},

1022 )

1023

1024 tokenizer = AutoTokenizer.from_pretrained(model_name,

[Bugfix] Fix modality limits in vision language example (#17721)