A high-throughput and memory-efficient inference and serving engine for LLMs
[Perf] Use bisect for mm feature lookup in model runner v2 (#45566)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Jun 14, 2026 at 00:22 UTC
e2bf2b3d8475715f1b951b6e9f4020af2721db6e
1 file changed
+8
-10
vllm/v1/worker/gpu/mm/encoder_runner.py
+8
-10
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.model_executor.models.interfaces import SupportsMultiModal
7
from vllm.multimodal.inputs import MultiModalKwargsItem
8
-from vllm.multimodal.utils import group_and_batch_mm_kwargs
8
+from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
9
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
10
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
11
@@ -91,19 +91,17 @@ class EncoderRunner:
91
continue
92
93
mm_features = self.encoder_cache.mm_features[req_id]
94
- for mm_feature in mm_features:
94
+ lo, hi = get_mm_features_in_window(
95
+ mm_features,
96
+ start=query_start[i],
97
+ end=query_end[i],
98
+ )
99
+ for idx in range(lo, hi):
100
+ mm_feature = mm_features[idx]
101
pos_info = mm_feature.mm_position
102
start_pos = pos_info.offset
103
num_encoder_tokens = pos_info.length
104
99
- if start_pos >= query_end[i]:
100
- # The encoder output is not needed in this step.
101
- break
102
- if start_pos + num_encoder_tokens <= query_start[i]:
103
- # The encoder output is already processed and stored
104
- # in the decoder's KV cache.
105
- continue
106
-
105
start_idx = max(query_start[i] - start_pos, 0)
106
end_idx = min(query_end[i] - start_pos, num_encoder_tokens)
107
assert start_idx < end_idx