@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d06e824006
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Set enforce_eager automatically for mllama (#12127)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Jan 17, 2025 at 04:30 UTC d06e824006d1ba4b92871347738ce1b89f658499
3 files changed +5 -5
examples/offline_inference/vision_language.py
-1
@@ -325,7 +325,6 @@ def run_mllama(question: str, modality: str):
325 model=model_name,
326 max_model_len=4096,
327 max_num_seqs=16,
328 - enforce_eager=True,
328 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
329 )
330
examples/offline_inference/vision_language_multi_image.py
-1
@@ -186,7 +186,6 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
186 model=model_name,
187 max_model_len=4096,
188 max_num_seqs=16,
189 - enforce_eager=True,
189 limit_mm_per_prompt={"image": len(image_urls)},
190 )
191
vllm/config.py
+5 -3
@@ -607,10 +607,12 @@ class ModelConfig:
607 self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
608 self.max_model_len)
609
610 - if (self.hf_config.model_type == 'deepseek_v3'
610 + MODEL_NOT_SUPPORT_CUDA_GRAPH = ['deepseek_v3', 'mllama']
611 + if (self.hf_config.model_type in MODEL_NOT_SUPPORT_CUDA_GRAPH
612 and not self.enforce_eager):
612 - logger.warning("CUDA graph is not supported for Deepseek V3 yet, "
613 - "fallback to the eager mode.")
613 + logger.warning(
614 + "CUDA graph is not supported for %s yet, fallback to the eager "
615 + "mode.", self.hf_config.model_type)
616 self.enforce_eager = True
617
618 def _verify_bnb_config(self) -> None: