@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48312e579a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 17:45 UTC

[Misc] Make `PlaceholderRange.get_num_embeds` a method (#34035)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 7, 2026 at 13:30 UTC 48312e579a6de16b3d15a40796ef165ed141c8dc

9 files changed +11 -12

tests/models/multimodal/processing/test_mllama4.py

+1 -1

@@ -48,7 +48,7 @@ def test_profiling(model_id: str, max_model_len: int):

48 ) # image start, image, image end

50 assert total_num_patches == sum(

51 - item.get_num_embeds for item in mm_inputs["mm_placeholders"]["image"]

51 + item.get_num_embeds() for item in mm_inputs["mm_placeholders"]["image"]

52 )

53 assert total_tokens == sum(

54 placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]

tests/multimodal/test_inputs.py

+1 -1

@@ -19,7 +19,7 @@ from vllm.multimodal.inputs import PlaceholderRange

19 def test_placeholder_range_get_num_embeds(is_embed, expected):

20 length = len(is_embed) if is_embed is not None else 5

21 pr = PlaceholderRange(offset=0, length=length, is_embed=is_embed)

22 - assert pr.get_num_embeds == expected

22 + assert pr.get_num_embeds() == expected

25 @pytest.mark.parametrize(

tests/v1/core/test_encoder_cache_manager.py

+4 -4

@@ -187,7 +187,7 @@ def test_schedule_request_multi_images_respect_compute_limit():

187 def test_encoder_cache_with_is_embed_mask():

188 class MockRequestWithMask(MockRequest):

189 def get_num_encoder_embeds(self, input_id: int) -> int:

190 - return self.mm_features[input_id].mm_position.get_num_embeds

190 + return self.mm_features[input_id].mm_position.get_num_embeds()

191

192 is_embed = torch.zeros(100, dtype=torch.bool)

193 is_embed[torch.tensor([5, 15, 25, 35, 45, 55, 65, 75])] = True

@@ -207,7 +207,7 @@ def test_encoder_cache_with_is_embed_mask():

207 assert "img1" in manager.cached

208

209 old_size = 100

210 - new_size = request.mm_features[0].mm_position.get_num_embeds

210 + new_size = request.mm_features[0].mm_position.get_num_embeds()

211 assert new_size == 8

212 savings_ratio = old_size / new_size

213 assert savings_ratio == 12.5

@@ -216,7 +216,7 @@ def test_encoder_cache_with_is_embed_mask():

216 def test_encoder_cache_mask_based_retrieval():

217 class MockRequestWithMask(MockRequest):

218 def get_num_encoder_embeds(self, input_id: int) -> int:

219 - return self.mm_features[input_id].mm_position.get_num_embeds

219 + return self.mm_features[input_id].mm_position.get_num_embeds()

220

221 is_embed = torch.tensor(

222 [False, False, True, True, False, True, True, True, False, False]

@@ -233,7 +233,7 @@ def test_encoder_cache_mask_based_retrieval():

233 manager = EncoderCacheManager(cache_size=50)

234 manager.allocate(request, 0)

235

236 - assert request.mm_features[0].mm_position.get_num_embeds == 5

236 + assert request.mm_features[0].mm_position.get_num_embeds() == 5

237

238 start_idx = 2

239 end_idx = 8

vllm/multimodal/budget.py

+1 -1

@@ -33,7 +33,7 @@ def get_mm_max_toks_per_item(

33 )

35 return {

36 - modality: sum(item.get_num_embeds for item in placeholders)

36 + modality: sum(item.get_num_embeds() for item in placeholders)

37 for modality, placeholders in mm_inputs["mm_placeholders"].items()

38 }

vllm/multimodal/inputs.py

-1

@@ -199,7 +199,6 @@ class PlaceholderRange:

199 def embeds_cumsum(self) -> torch.Tensor | None:

200 return None if self.is_embed is None else self.is_embed.cumsum(dim=0)

201

202 - @cached_property

202 def get_num_embeds(self) -> int:

203 if self.embeds_cumsum is None:

204 return self.length

vllm/v1/core/sched/scheduler.py

+1 -1

@@ -1100,7 +1100,7 @@ class Scheduler(SchedulerInterface):

1100 for i, mm_feature in enumerate(mm_features):

1101 start_pos = mm_feature.mm_position.offset

1102 num_encoder_tokens = mm_feature.mm_position.length

1103 - num_encoder_embeds = mm_feature.mm_position.get_num_embeds

1103 + num_encoder_embeds = mm_feature.mm_position.get_num_embeds()

1104 item_identifier = mm_feature.identifier

1105

1106 # The encoder output is needed if the two ranges overlap:

vllm/v1/engine/input_processor.py

+1 -1

@@ -786,7 +786,7 @@ class InputProcessor:

786 decoder_mm_positions = prompt_inputs["mm_placeholders"]

787 for modality, mm_positions in decoder_mm_positions.items():

788 for mm_position in mm_positions:

789 - embed_length = mm_position.get_num_embeds

789 + embed_length = mm_position.get_num_embeds()

790 if embed_length > self.mm_encoder_cache_size:

791 raise ValueError(

792 f"The {prompt_type} prompt contains a(n) {modality} item "

vllm/v1/request.py

+1 -1

@@ -260,7 +260,7 @@ class Request:

260

261 def get_num_encoder_embeds(self, input_id: int) -> int:

262 assert input_id < len(self.mm_features)

263 - return self.mm_features[input_id].mm_position.get_num_embeds

263 + return self.mm_features[input_id].mm_position.get_num_embeds()

264

265 def record_event(

266 self,

vllm/v1/worker/gpu_model_runner.py

+1 -1

@@ -2326,7 +2326,7 @@ class GPUModelRunner(

2326

2327 # Prefer pos_info.get_num_embeds to count precise MM embedding tokens.

2328 num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]

2329 - pos_info.get_num_embeds

2329 + pos_info.get_num_embeds()

2330 )

2331 prompt_lora_mapping.append(lora_id)

2332 token_lora_mapping.extend([lora_id] * num_tokens)