A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][CI] Gemma3 Transformers multimodal encoder profiling and build prompt-embedding fixtures (#44952)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 9, 2026 at 00:49 UTC
f843ac1a1c4f0d69a19a086c65603db75edebac7
2 files changed
+35
-2
tests/entrypoints/openai/chat_completion/test_chat_completion_with_prompt_embeds.py
+6
-1
@@ -14,6 +14,7 @@ import torch
14
from openai import BadRequestError
15
16
from tests.utils import VLLM_PATH, RemoteOpenAIServer
17
+from vllm.platforms import current_platform
18
19
MODEL_NAME = "facebook/opt-125m"
20
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
@@ -41,7 +42,11 @@ def server_args() -> list[str]:
42
43
44
@pytest.fixture(scope="module")
44
-def server(server_args):
45
+def server(server_args, request):
46
+ if current_platform.is_rocm():
47
+ # Materialize HF embeddings before the server reserves ROCm VRAM.
48
+ request.getfixturevalue("prompt_embeds_b64")
49
+ request.getfixturevalue("aligned_content_and_embeds_b64")
50
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
51
yield remote_server
52
vllm/model_executor/models/transformers/multimodal.py
+29
-1
@@ -72,7 +72,35 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
72
image_sizes=([height, width],), **mm_processor_kwargs
73
)
74
image_tokens = mm_tokens["num_image_tokens"][0]
75
- return image_tokens
75
+ return self._get_max_encoder_tokens(processor, mm_tokens) or image_tokens
76
+
77
+ @staticmethod
78
+ def _get_mm_values(mm_tokens: object, key: str) -> object:
79
+ if isinstance(mm_tokens, Mapping):
80
+ return mm_tokens.get(key)
81
+ return getattr(mm_tokens, key, None)
82
+
83
+ def _get_max_encoder_tokens(
84
+ self, processor: object, mm_tokens: object
85
+ ) -> int | None:
86
+ if "gemma3" not in processor.__class__.__name__.lower():
87
+ return None
88
+
89
+ vision_config = getattr(self.get_hf_config(), "vision_config", None)
90
+ image_size = getattr(vision_config, "image_size", None)
91
+ patch_size = getattr(vision_config, "patch_size", None)
92
+ if not image_size or not patch_size:
93
+ return None
94
+
95
+ # Gemma3 pools each 64x64 SigLIP patch grid down to 256 image tokens.
96
+ # Profile the vision encoder against the pre-pooling patch-token count.
97
+ patches_per_image = (image_size // patch_size) ** 2
98
+ num_image_patches = self._get_mm_values(mm_tokens, "num_image_patches") or [1]
99
+ if isinstance(num_image_patches, int):
100
+ max_image_patches = num_image_patches
101
+ else:
102
+ max_image_patches = max(num_image_patches)
103
+ return patches_per_image * int(max_image_patches)
104
105
def get_max_image_size(self):
106
return 10_000, 10_000 # hardcode for arbitrary very large size