A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Doc] Add example for Step3-VL (#22061)
Signed-off-by: Roger Wang <hey@rogerw.me>
Roger Wang committed
Aug 1, 2025 at 01:35 UTC
27a145e8931582fc74c1f46e0e4630c610b96160
2 files changed
+270
-211
examples/offline_inference/vision_language.py
+157
-127
@@ -423,32 +423,6 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
423
)
424
425
426
-# SmolVLM2-2.2B-Instruct
427
-def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
428
- assert modality == "image"
429
- model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
430
-
431
- engine_args = EngineArgs(
432
- model=model_name,
433
- max_model_len=8192,
434
- max_num_seqs=2,
435
- enforce_eager=True,
436
- mm_processor_kwargs={
437
- "max_image_size": {"longest_edge": 384},
438
- },
439
- limit_mm_per_prompt={modality: 1},
440
- )
441
- prompts = [
442
- (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
443
- for question in questions
444
- ]
445
-
446
- return ModelRequestData(
447
- engine_args=engine_args,
448
- prompts=prompts,
449
- )
450
-
451
-
426
# Intern-S1
427
def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
428
model_name = "internlm/Intern-S1"
@@ -522,44 +496,6 @@ def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
496
)
497
498
525
-# Nemontron_VL
526
-def run_nemotron_vl(questions: list[str], modality: str) -> ModelRequestData:
527
- model_name = "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"
528
-
529
- engine_args = EngineArgs(
530
- model=model_name,
531
- trust_remote_code=True,
532
- max_model_len=8192,
533
- limit_mm_per_prompt={modality: 1},
534
- )
535
-
536
- assert modality == "image"
537
- placeholder = "<image>"
538
-
539
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
540
- messages = [
541
- [{"role": "user", "content": f"{placeholder}\n{question}"}]
542
- for question in questions
543
- ]
544
- prompts = tokenizer.apply_chat_template(
545
- messages, tokenize=False, add_generation_prompt=True
546
- )
547
-
548
- # Stop tokens for InternVL
549
- # models variants may have different stop tokens
550
- # please refer to the model card for the correct "stop words":
551
- # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
552
- stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
553
- stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
554
- stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
555
-
556
- return ModelRequestData(
557
- engine_args=engine_args,
558
- prompts=prompts,
559
- stop_token_ids=stop_token_ids,
560
- )
561
-
562
-
499
# Keye-VL
500
def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:
501
model_name = "Kwai-Keye/Keye-VL-8B-Preview"
@@ -615,6 +551,41 @@ def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
551
)
552
553
554
+def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
555
+ assert modality == "image"
556
+
557
+ model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
558
+
559
+ engine_args = EngineArgs(
560
+ model=model_name,
561
+ max_model_len=8192,
562
+ max_num_seqs=4,
563
+ tensor_parallel_size=8,
564
+ gpu_memory_utilization=0.4,
565
+ limit_mm_per_prompt={modality: 1},
566
+ )
567
+
568
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
569
+ messages = [
570
+ [
571
+ {
572
+ "role": "user",
573
+ "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
574
+ }
575
+ ]
576
+ for question in questions
577
+ ]
578
+ prompts = tokenizer.apply_chat_template(
579
+ messages, add_generation_prompt=True, tokenize=False
580
+ )
581
+ stop_token_ids = None
582
+ return ModelRequestData(
583
+ engine_args=engine_args,
584
+ prompts=prompts,
585
+ stop_token_ids=stop_token_ids,
586
+ )
587
+
588
+
589
# LLaVA-1.5
590
def run_llava(questions: list[str], modality: str) -> ModelRequestData:
591
assert modality == "image"
@@ -857,63 +828,66 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
828
)
829
830
860
-def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
831
+# Molmo
832
+def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
833
assert modality == "image"
834
863
- model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
835
+ model_name = "allenai/Molmo-7B-D-0924"
836
837
engine_args = EngineArgs(
838
model=model_name,
867
- max_model_len=8192,
868
- max_num_seqs=4,
869
- tensor_parallel_size=8,
870
- gpu_memory_utilization=0.4,
839
+ trust_remote_code=True,
840
+ dtype="bfloat16",
841
limit_mm_per_prompt={modality: 1},
842
)
843
874
- tokenizer = AutoTokenizer.from_pretrained(model_name)
875
- messages = [
876
- [
877
- {
878
- "role": "user",
879
- "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
880
- }
881
- ]
844
+ prompts = [
845
+ f"<|im_start|>user <image>\n{question}<|im_end|> \
846
+ <|im_start|>assistant\n"
847
for question in questions
848
]
884
- prompts = tokenizer.apply_chat_template(
885
- messages, add_generation_prompt=True, tokenize=False
886
- )
887
- stop_token_ids = None
849
+
850
return ModelRequestData(
851
engine_args=engine_args,
852
prompts=prompts,
891
- stop_token_ids=stop_token_ids,
853
)
854
855
895
-# Molmo
896
-def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
897
- assert modality == "image"
898
-
899
- model_name = "allenai/Molmo-7B-D-0924"
856
+# Nemontron_VL
857
+def run_nemotron_vl(questions: list[str], modality: str) -> ModelRequestData:
858
+ model_name = "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"
859
860
engine_args = EngineArgs(
861
model=model_name,
862
trust_remote_code=True,
904
- dtype="bfloat16",
863
+ max_model_len=8192,
864
limit_mm_per_prompt={modality: 1},
865
)
866
908
- prompts = [
909
- f"<|im_start|>user <image>\n{question}<|im_end|> \
910
- <|im_start|>assistant\n"
867
+ assert modality == "image"
868
+ placeholder = "<image>"
869
+
870
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
871
+ messages = [
872
+ [{"role": "user", "content": f"{placeholder}\n{question}"}]
873
for question in questions
874
]
875
+ prompts = tokenizer.apply_chat_template(
876
+ messages, tokenize=False, add_generation_prompt=True
877
+ )
878
+
879
+ # Stop tokens for InternVL
880
+ # models variants may have different stop tokens
881
+ # please refer to the model card for the correct "stop words":
882
+ # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
883
+ stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
884
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
885
+ stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
886
887
return ModelRequestData(
888
engine_args=engine_args,
889
prompts=prompts,
890
+ stop_token_ids=stop_token_ids,
891
)
892
893
@@ -1274,10 +1248,11 @@ def run_qwen2_5_omni(questions: list[str], modality: str):
1248
)
1249
1250
1277
-# omni-research/Tarsier-7b
1278
-def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
1251
+# SkyworkR1V
1252
+def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
1253
assert modality == "image"
1280
- model_name = "omni-research/Tarsier-7b"
1254
+
1255
+ model_name = "Skywork/Skywork-R1V-38B"
1256
1257
engine_args = EngineArgs(
1258
model=model_name,
@@ -1285,36 +1260,73 @@ def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
1260
max_model_len=4096,
1261
limit_mm_per_prompt={modality: 1},
1262
)
1288
- prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
1263
+
1264
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1265
+ messages = [
1266
+ [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
1267
+ ]
1268
+ prompts = tokenizer.apply_chat_template(
1269
+ messages, tokenize=False, add_generation_prompt=True
1270
+ )
1271
+
1272
+ # Stop tokens for SkyworkR1V
1273
+ # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
1274
+ stop_tokens = ["<｜end▁of▁sentence｜>", "<|endoftext|>"]
1275
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1276
1277
return ModelRequestData(
1278
engine_args=engine_args,
1279
prompts=prompts,
1280
+ stop_token_ids=stop_token_ids,
1281
)
1282
1283
1296
-def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
1297
- model_name = "omni-research/Tarsier2-Recap-7b"
1284
+# SmolVLM2-2.2B-Instruct
1285
+def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
1286
+ assert modality == "image"
1287
+ model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
1288
1289
engine_args = EngineArgs(
1290
model=model_name,
1301
- max_model_len=4096,
1302
- hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
1291
+ max_model_len=8192,
1292
+ max_num_seqs=2,
1293
+ enforce_eager=True,
1294
+ mm_processor_kwargs={
1295
+ "max_image_size": {"longest_edge": 384},
1296
+ },
1297
limit_mm_per_prompt={modality: 1},
1298
)
1299
+ prompts = [
1300
+ (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
1301
+ for question in questions
1302
+ ]
1303
1306
- if modality == "image":
1307
- placeholder = "<|image_pad|>"
1308
- elif modality == "video":
1309
- placeholder = "<|video_pad|>"
1304
+ return ModelRequestData(
1305
+ engine_args=engine_args,
1306
+ prompts=prompts,
1307
+ )
1308
+
1309
+
1310
+# Step3
1311
+def run_step3(questions: list[str], modality: str) -> ModelRequestData:
1312
+ assert modality == "image"
1313
+
1314
+ model_name = "stepfun-ai/step3-fp8"
1315
+
1316
+ # NOTE: Below are verified configurations for step3-fp8
1317
+ # on 8xH100 GPUs.
1318
+ engine_args = EngineArgs(
1319
+ model=model_name,
1320
+ max_num_batched_tokens=4096,
1321
+ gpu_memory_utilization=0.85,
1322
+ tensor_parallel_size=8,
1323
+ limit_mm_per_prompt={modality: 1},
1324
+ reasoning_parser="step3",
1325
+ )
1326
1327
prompts = [
1312
- (
1313
- "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
1314
- f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
1315
- f"{question}<|im_end|>\n"
1316
- "<|im_start|>assistant\n"
1317
- )
1328
+ "<｜begin▁of▁sentence｜> You are a helpful assistant. <|BOT|>user\n "
1329
+ f"<im_patch>{question} <|EOT|><|BOT|>assistant\n<think>\n"
1330
for question in questions
1331
]
1332
@@ -1324,11 +1336,10 @@ def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
1336
)
1337
1338
1327
-# SkyworkR1V
1328
-def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
1339
+# omni-research/Tarsier-7b
1340
+def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
1341
assert modality == "image"
1330
-
1331
- model_name = "Skywork/Skywork-R1V-38B"
1342
+ model_name = "omni-research/Tarsier-7b"
1343
1344
engine_args = EngineArgs(
1345
model=model_name,
@@ -1336,24 +1347,42 @@ def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
1347
max_model_len=4096,
1348
limit_mm_per_prompt={modality: 1},
1349
)
1350
+ prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
1351
1340
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1341
- messages = [
1342
- [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
1343
- ]
1344
- prompts = tokenizer.apply_chat_template(
1345
- messages, tokenize=False, add_generation_prompt=True
1352
+ return ModelRequestData(
1353
+ engine_args=engine_args,
1354
+ prompts=prompts,
1355
)
1356
1348
- # Stop tokens for SkyworkR1V
1349
- # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
1350
- stop_tokens = ["<｜end▁of▁sentence｜>", "<|endoftext|>"]
1351
- stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1357
+
1358
+def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
1359
+ model_name = "omni-research/Tarsier2-Recap-7b"
1360
+
1361
+ engine_args = EngineArgs(
1362
+ model=model_name,
1363
+ max_model_len=4096,
1364
+ hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
1365
+ limit_mm_per_prompt={modality: 1},
1366
+ )
1367
+
1368
+ if modality == "image":
1369
+ placeholder = "<|image_pad|>"
1370
+ elif modality == "video":
1371
+ placeholder = "<|video_pad|>"
1372
+
1373
+ prompts = [
1374
+ (
1375
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
1376
+ f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
1377
+ f"{question}<|im_end|>\n"
1378
+ "<|im_start|>assistant\n"
1379
+ )
1380
+ for question in questions
1381
+ ]
1382
1383
return ModelRequestData(
1384
engine_args=engine_args,
1385
prompts=prompts,
1356
- stop_token_ids=stop_token_ids,
1386
)
1387
1388
@@ -1373,9 +1402,9 @@ model_example_map = {
1402
"idefics3": run_idefics3,
1403
"interns1": run_interns1,
1404
"internvl_chat": run_internvl,
1376
- "nemotron_vl": run_nemotron_vl,
1405
"keye_vl": run_keye_vl,
1406
"kimi_vl": run_kimi_vl,
1407
+ "llama4": run_llama4,
1408
"llava": run_llava,
1409
"llava-next": run_llava_next,
1410
"llava-next-video": run_llava_next_video,
@@ -1385,8 +1414,8 @@ model_example_map = {
1414
"minicpmv": run_minicpmv,
1415
"mistral3": run_mistral3,
1416
"mllama": run_mllama,
1388
- "llama4": run_llama4,
1417
"molmo": run_molmo,
1418
+ "nemotron_vl": run_nemotron_vl,
1419
"NVLM_D": run_nvlm_d,
1420
"ovis": run_ovis,
1421
"paligemma": run_paligemma,
@@ -1401,6 +1430,7 @@ model_example_map = {
1430
"qwen2_5_omni": run_qwen2_5_omni,
1431
"skywork_chat": run_skyworkr1v,
1432
"smolvlm": run_smolvlm,
1433
+ "step3": run_step3,
1434
"tarsier": run_tarsier,
1435
"tarsier2": run_tarsier2,
1436
}
examples/offline_inference/vision_language_multi_image.py
+113
-84
@@ -197,36 +197,55 @@ def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
197
)
198
199
200
-def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
201
- model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
200
+def load_hyperclovax_seed_vision(
201
+ question: str, image_urls: list[str]
202
+) -> ModelRequestData:
203
+ model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
204
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
205
203
- # The configuration below has been confirmed to launch on a single L40 GPU.
206
engine_args = EngineArgs(
207
model=model_name,
206
- max_model_len=8192,
207
- max_num_seqs=16,
208
- enforce_eager=True,
208
+ trust_remote_code=True,
209
+ max_model_len=16384,
210
limit_mm_per_prompt={"image": len(image_urls)},
210
- # if you are running out of memory, you can reduce the "longest_edge".
211
- # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
212
- mm_processor_kwargs={
213
- "size": {"longest_edge": 2 * 364},
214
- },
211
)
212
217
- placeholders = "\n".join(
218
- f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
213
+ message = {"role": "user", "content": list()}
214
+ for _image_url in image_urls:
215
+ message["content"].append(
216
+ {
217
+ "type": "image",
218
+ "image": _image_url,
219
+ "ocr": "",
220
+ "lens_keywords": "",
221
+ "lens_local_keywords": "",
222
+ }
223
+ )
224
+ message["content"].append(
225
+ {
226
+ "type": "text",
227
+ "text": question,
228
+ }
229
)
220
- prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
230
+
231
+ prompt = tokenizer.apply_chat_template(
232
+ [
233
+ message,
234
+ ],
235
+ tokenize=False,
236
+ add_generation_prompt=True,
237
+ )
238
+
239
return ModelRequestData(
240
engine_args=engine_args,
241
prompt=prompt,
242
+ stop_token_ids=None,
243
image_data=[fetch_image(url) for url in image_urls],
244
)
245
246
228
-def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
229
- model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
247
+def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
248
+ model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
249
250
# The configuration below has been confirmed to launch on a single L40 GPU.
251
engine_args = EngineArgs(
@@ -235,17 +254,17 @@ def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
254
max_num_seqs=16,
255
enforce_eager=True,
256
limit_mm_per_prompt={"image": len(image_urls)},
257
+ # if you are running out of memory, you can reduce the "longest_edge".
258
+ # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
259
mm_processor_kwargs={
239
- "max_image_size": {"longest_edge": 384},
260
+ "size": {"longest_edge": 2 * 364},
261
},
262
)
263
264
placeholders = "\n".join(
265
f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
266
)
246
- prompt = (
247
- f"<|im_start|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
248
- )
267
+ prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
268
return ModelRequestData(
269
engine_args=engine_args,
270
prompt=prompt,
@@ -316,60 +335,13 @@ def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
335
)
336
337
319
-def load_hyperclovax_seed_vision(
320
- question: str, image_urls: list[str]
321
-) -> ModelRequestData:
322
- model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
323
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
324
-
325
- engine_args = EngineArgs(
326
- model=model_name,
327
- trust_remote_code=True,
328
- max_model_len=16384,
329
- limit_mm_per_prompt={"image": len(image_urls)},
330
- )
331
-
332
- message = {"role": "user", "content": list()}
333
- for _image_url in image_urls:
334
- message["content"].append(
335
- {
336
- "type": "image",
337
- "image": _image_url,
338
- "ocr": "",
339
- "lens_keywords": "",
340
- "lens_local_keywords": "",
341
- }
342
- )
343
- message["content"].append(
344
- {
345
- "type": "text",
346
- "text": question,
347
- }
348
- )
349
-
350
- prompt = tokenizer.apply_chat_template(
351
- [
352
- message,
353
- ],
354
- tokenize=False,
355
- add_generation_prompt=True,
356
- )
357
-
358
- return ModelRequestData(
359
- engine_args=engine_args,
360
- prompt=prompt,
361
- stop_token_ids=None,
362
- image_data=[fetch_image(url) for url in image_urls],
363
- )
364
-
338
+def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
339
+ model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
340
366
-def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
367
- # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
368
- # it will generate poor response for multi-image inputs!
369
- model_name = "llava-hf/llava-1.5-7b-hf"
341
engine_args = EngineArgs(
342
model=model_name,
372
- max_num_seqs=16,
343
+ max_model_len=131072,
344
+ tensor_parallel_size=8,
345
limit_mm_per_prompt={"image": len(image_urls)},
346
)
347
@@ -397,11 +369,12 @@ def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
369
)
370
371
400
-def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
401
- model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
372
+def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
373
+ # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
374
+ # it will generate poor response for multi-image inputs!
375
+ model_name = "llava-hf/llava-1.5-7b-hf"
376
engine_args = EngineArgs(
377
model=model_name,
404
- max_model_len=8192,
378
max_num_seqs=16,
379
limit_mm_per_prompt={"image": len(image_urls)},
380
)
@@ -430,11 +403,11 @@ def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
403
)
404
405
433
-def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestData:
434
- model_name = "llava-hf/llava-onevision-qwen2-7b-ov-hf"
406
+def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
407
+ model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
408
engine_args = EngineArgs(
409
model=model_name,
437
- max_model_len=16384,
410
+ max_model_len=8192,
411
max_num_seqs=16,
412
limit_mm_per_prompt={"image": len(image_urls)},
413
)
@@ -463,13 +436,12 @@ def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestDa
436
)
437
438
466
-def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
467
- model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
468
-
439
+def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestData:
440
+ model_name = "llava-hf/llava-onevision-qwen2-7b-ov-hf"
441
engine_args = EngineArgs(
442
model=model_name,
471
- max_model_len=131072,
472
- tensor_parallel_size=8,
443
+ max_model_len=16384,
444
+ max_num_seqs=16,
445
limit_mm_per_prompt={"image": len(image_urls)},
446
)
447
@@ -954,6 +926,62 @@ def load_qwen2_5_vl(question: str, image_urls: list[str]) -> ModelRequestData:
926
)
927
928
929
+def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
930
+ model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
931
+
932
+ # The configuration below has been confirmed to launch on a single L40 GPU.
933
+ engine_args = EngineArgs(
934
+ model=model_name,
935
+ max_model_len=8192,
936
+ max_num_seqs=16,
937
+ enforce_eager=True,
938
+ limit_mm_per_prompt={"image": len(image_urls)},
939
+ mm_processor_kwargs={
940
+ "max_image_size": {"longest_edge": 384},
941
+ },
942
+ )
943
+
944
+ placeholders = "\n".join(
945
+ f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
946
+ )
947
+ prompt = (
948
+ f"<|im_start|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
949
+ )
950
+ return ModelRequestData(
951
+ engine_args=engine_args,
952
+ prompt=prompt,
953
+ image_data=[fetch_image(url) for url in image_urls],
954
+ )
955
+
956
+
957
+def load_step3(question: str, image_urls: list[str]) -> ModelRequestData:
958
+ model_name = "stepfun-ai/step3-fp8"
959
+
960
+ # NOTE: Below are verified configurations for step3-fp8
961
+ # on 8xH100 GPUs.
962
+ engine_args = EngineArgs(
963
+ model=model_name,
964
+ max_num_batched_tokens=4096,
965
+ gpu_memory_utilization=0.85,
966
+ tensor_parallel_size=8,
967
+ limit_mm_per_prompt={"image": len(image_urls)},
968
+ reasoning_parser="step3",
969
+ )
970
+
971
+ prompt = (
972
+ "<｜begin▁of▁sentence｜> You are a helpful assistant. <|BOT|>user\n "
973
+ f"{'<im_patch>' * len(image_urls)}{question} <|EOT|><|BOT|"
974
+ ">assistant\n<think>\n"
975
+ )
976
+ image_data = [fetch_image(url) for url in image_urls]
977
+
978
+ return ModelRequestData(
979
+ engine_args=engine_args,
980
+ prompt=prompt,
981
+ image_data=image_data,
982
+ )
983
+
984
+
985
def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:
986
model_name = "omni-research/Tarsier-7b"
987
@@ -1006,16 +1034,16 @@ model_example_map = {
1034
"deepseek_vl_v2": load_deepseek_vl2,
1035
"gemma3": load_gemma3,
1036
"h2ovl_chat": load_h2ovl,
1037
+ "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
1038
"idefics3": load_idefics3,
1039
"interns1": load_interns1,
1040
"internvl_chat": load_internvl,
1012
- "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
1041
"keye_vl": load_keye_vl,
1042
"kimi_vl": load_kimi_vl,
1043
+ "llama4": load_llama4,
1044
"llava": load_llava,
1045
"llava-next": load_llava_next,
1046
"llava-onevision": load_llava_onevision,
1018
- "llama4": load_llama4,
1047
"mistral3": load_mistral3,
1048
"mllama": load_mllama,
1049
"NVLM_D": load_nvlm_d,
@@ -1028,6 +1056,7 @@ model_example_map = {
1056
"qwen2_vl": load_qwen2_vl,
1057
"qwen2_5_vl": load_qwen2_5_vl,
1058
"smolvlm": load_smolvlm,
1059
+ "step3": load_step3,
1060
"tarsier": load_tarsier,
1061
"tarsier2": load_tarsier2,
1062
}