A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Model] Remove image mm limit for LLaMa4 (#16365)
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com>
Ye (Charlotte) Qi committed
Apr 10, 2025 at 02:36 UTC
61de3ef74b9cd5594e1d79ba573258adf481382c
2 files changed
+26
-7
examples/offline_inference/vision_language_multi_image.py
+23
-6
@@ -22,6 +22,16 @@ QUESTION = "What is the content of each image?"
22
IMAGE_URLS = [
23
"https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg",
24
"https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg",
25
+ "https://upload.wikimedia.org/wikipedia/commons/2/26/Ultramarine_Flycatcher_%28Ficedula_superciliaris%29_Naggar%2C_Himachal_Pradesh%2C_2013_%28cropped%29.JPG",
26
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e5/Anim1754_-_Flickr_-_NOAA_Photo_Library_%281%29.jpg/2560px-Anim1754_-_Flickr_-_NOAA_Photo_Library_%281%29.jpg",
27
+ "https://upload.wikimedia.org/wikipedia/commons/d/d4/Starfish%2C_Caswell_Bay_-_geograph.org.uk_-_409413.jpg",
28
+ "https://upload.wikimedia.org/wikipedia/commons/6/69/Grapevinesnail_01.jpg",
29
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0b/Texas_invasive_Musk_Thistle_1.jpg/1920px-Texas_invasive_Musk_Thistle_1.jpg",
30
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/Huskiesatrest.jpg/2880px-Huskiesatrest.jpg",
31
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/6/68/Orange_tabby_cat_sitting_on_fallen_leaves-Hisashi-01A.jpg/1920px-Orange_tabby_cat_sitting_on_fallen_leaves-Hisashi-01A.jpg",
32
+ "https://upload.wikimedia.org/wikipedia/commons/3/30/George_the_amazing_guinea_pig.jpg",
33
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1f/Oryctolagus_cuniculus_Rcdo.jpg/1920px-Oryctolagus_cuniculus_Rcdo.jpg",
34
+ "https://upload.wikimedia.org/wikipedia/commons/9/98/Horse-and-pony.jpg",
35
]
36
37
@@ -285,8 +295,7 @@ def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
295
296
engine_args = EngineArgs(
297
model=model_name,
288
- max_model_len=8192,
289
- max_num_seqs=4,
298
+ max_model_len=131072,
299
tensor_parallel_size=8,
300
limit_mm_per_prompt={"image": len(image_urls)},
301
)
@@ -660,7 +669,7 @@ def run_generate(model, question: str, image_urls: list[str],
669
llm.llm_engine.add_lora(lora_request=lora_request)
670
671
sampling_params = SamplingParams(temperature=0.0,
663
- max_tokens=128,
672
+ max_tokens=256,
673
stop_token_ids=req_data.stop_token_ids)
674
675
outputs = llm.generate(
@@ -694,7 +703,7 @@ def run_chat(model: str, question: str, image_urls: list[str],
703
llm.llm_engine.add_lora(lora_request=lora_request)
704
705
sampling_params = SamplingParams(temperature=0.0,
697
- max_tokens=128,
706
+ max_tokens=256,
707
stop_token_ids=req_data.stop_token_ids)
708
outputs = llm.chat(
709
[{
@@ -729,10 +738,12 @@ def main(args: Namespace):
738
method = args.method
739
seed = args.seed
740
741
+ image_urls = IMAGE_URLS[:args.num_images]
742
+
743
if method == "generate":
733
- run_generate(model, QUESTION, IMAGE_URLS, seed)
744
+ run_generate(model, QUESTION, image_urls, seed)
745
elif method == "chat":
735
- run_chat(model, QUESTION, IMAGE_URLS, seed)
746
+ run_chat(model, QUESTION, image_urls, seed)
747
else:
748
raise ValueError(f"Invalid method: {method}")
749
@@ -757,6 +768,12 @@ if __name__ == "__main__":
768
type=int,
769
default=None,
770
help="Set the seed when initializing `vllm.LLM`.")
771
+ parser.add_argument(
772
+ "--num-images",
773
+ "-n",
774
+ choices=list(range(1, 13)), # 12 is the max number of images
775
+ default=2,
776
+ help="Number of images to use for the demo.")
777
778
args = parser.parse_args()
779
main(args)
vllm/model_executor/models/mllama4.py
+3
-1
@@ -477,7 +477,9 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
477
**kwargs)
478
479
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
480
- return {"image": 10}
480
+ # Although vLLM can support more images from an infra capability
481
+ # perspective, we do not recommend using >10 images in practice.
482
+ return {"image": None}
483
484
@staticmethod
485
def get_patch_per_chunk(vision_config: Llama4VisionConfig) -> int: