A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Set enforce_eager automatically for mllama (#12127)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Jan 17, 2025 at 04:30 UTC
d06e824006d1ba4b92871347738ce1b89f658499
3 files changed
+5
-5
examples/offline_inference/vision_language.py
-1
@@ -325,7 +325,6 @@ def run_mllama(question: str, modality: str):
325
model=model_name,
326
max_model_len=4096,
327
max_num_seqs=16,
328
- enforce_eager=True,
328
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
329
)
330
examples/offline_inference/vision_language_multi_image.py
-1
@@ -186,7 +186,6 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
186
model=model_name,
187
max_model_len=4096,
188
max_num_seqs=16,
189
- enforce_eager=True,
189
limit_mm_per_prompt={"image": len(image_urls)},
190
)
191
vllm/config.py
+5
-3
@@ -607,10 +607,12 @@ class ModelConfig:
607
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
608
self.max_model_len)
609
610
- if (self.hf_config.model_type == 'deepseek_v3'
610
+ MODEL_NOT_SUPPORT_CUDA_GRAPH = ['deepseek_v3', 'mllama']
611
+ if (self.hf_config.model_type in MODEL_NOT_SUPPORT_CUDA_GRAPH
612
and not self.enforce_eager):
612
- logger.warning("CUDA graph is not supported for Deepseek V3 yet, "
613
- "fallback to the eager mode.")
613
+ logger.warning(
614
+ "CUDA graph is not supported for %s yet, fallback to the eager "
615
+ "mode.", self.hf_config.model_type)
616
self.enforce_eager = True
617
618
def _verify_bnb_config(self) -> None: