A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
Enable CUDA graph support for llama 3.2 vision (#14917)
Signed-off-by: Matt Ritter <100659061+mritterfigma@users.noreply.github.com>
Matt Ritter committed
Mar 19, 2025 at 23:29 UTC
a8652f4f0f10a274ecdef3afbd9a607a997f5f21
3 files changed
+1
-13
tests/models/encoder_decoder/vision_language/test_mllama.py
-4
@@ -215,7 +215,6 @@ def _run_test(
215
max_num_seqs=2,
216
tensor_parallel_size=tensor_parallel_size,
217
distributed_executor_backend=distributed_executor_backend,
218
- enforce_eager=True,
218
limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
219
}) as vllm_model:
220
vllm_outputs_per_image = [
@@ -425,7 +424,6 @@ def test_bnb_regression(
424
dtype=dtype,
425
max_model_len=4096,
426
max_num_seqs=2,
428
- enforce_eager=True,
427
quantization="bitsandbytes",
428
load_format="bitsandbytes",
429
)
@@ -481,7 +479,6 @@ def test_explicit_implicit_prompt(
479
max_model_len=4096,
480
max_num_seqs=2,
481
tensor_parallel_size=1,
484
- enforce_eager=True,
482
)
483
sampling_params = SamplingParams(
484
temperature=0,
@@ -513,7 +510,6 @@ def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
510
max_model_len=4096,
511
max_num_seqs=2,
512
tensor_parallel_size=1,
516
- enforce_eager=True,
513
limit_mm_per_prompt={"image":
514
_LIMIT_IMAGE_PER_PROMPT}) as vllm_model:
515
vllm/config.py
-8
@@ -670,14 +670,6 @@ class ModelConfig:
670
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
671
self.max_model_len)
672
673
- MODEL_NOT_SUPPORT_CUDA_GRAPH = ['mllama']
674
- if (self.hf_config.model_type in MODEL_NOT_SUPPORT_CUDA_GRAPH
675
- and not self.enforce_eager):
676
- logger.warning(
677
- "CUDA graph is not supported for %s yet, fallback to the eager "
678
- "mode.", self.hf_config.model_type)
679
- self.enforce_eager = True
680
-
673
def _verify_bnb_config(self) -> None:
674
"""
675
The current version of bitsandbytes (0.44.0) with 8-bit models does not
vllm/model_executor/models/mllama.py
+1
-1
@@ -1368,7 +1368,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,
1368
full_text_row_masked_out_mask = (
1369
attn_metadata.encoder_seq_lens_tensor
1370
!= 0).reshape(-1, 1).to(input_ids.device)
1371
- skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1371
+ skip_cross_attention = attn_metadata.max_encoder_seq_len == 0
1372
1373
# For image-present prefill.
1374
else: