A high-throughput and memory-efficient inference and serving engine for LLMs
[ModelRunnerV2] Various model/config compatibility fixes (#45868)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Jun 16, 2026 at 20:23 UTC
14b438a98b0beb148c4bbe0ff126f2a32696aa92
4 files changed
+18
-4
vllm/config/vllm.py
+8
@@ -2026,6 +2026,14 @@ class VllmConfig:
2026
):
2027
unsupported.append("sequence parallelism")
2028
2029
+ # V2 does not implement the external_launcher (torchrun) PP-output
2030
+ # broadcast that V1 uses to keep all ranks in sync (broadcast_pp_output).
2031
+ if (
2032
+ self.parallel_config.distributed_executor_backend == "external_launcher"
2033
+ and self.parallel_config.pipeline_parallel_size > 1
2034
+ ):
2035
+ unsupported.append("pipeline parallelism with external_launcher")
2036
+
2037
if speculative_config is not None:
2038
# TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
2039
if speculative_config.method in ("ngram", "ngram_gpu"):
vllm/v1/worker/gpu/mm/encoder_runner.py
+1
-1
@@ -124,7 +124,7 @@ class EncoderRunner:
124
mm_embeds_item = encoder_output[start_idx:end_idx]
125
126
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
127
- is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = (
127
+ is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
128
True if is_embed is None else is_embed
129
)
130
mm_embeds.append(mm_embeds_item)
vllm/v1/worker/gpu/model_runner.py
+5
-1
@@ -1204,6 +1204,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1204
self.kv_cache_config,
1205
)
1206
1207
+ input_ids = input_batch.input_ids
1208
inputs_embeds = None
1209
if self.supports_mm_inputs and self.is_first_pp_rank:
1210
# Run MM encoder (if needed) and get multimodal embeddings.
@@ -1222,11 +1223,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1223
inputs_embeds = self.model_state.get_mm_embeddings(
1224
scheduler_output.scheduled_encoder_inputs, input_batch
1225
)
1226
+ if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
1227
+ input_ids = None
1228
1229
model_inputs = {
1227
- "input_ids": input_batch.input_ids,
1230
+ "input_ids": input_ids,
1231
"positions": input_batch.positions,
1232
"inputs_embeds": inputs_embeds,
1233
+ "intermediate_tensors": None,
1234
# NOTE: Values returned by `prepare_inputs` will override the default
1235
# values above.
1236
**self.model_state.prepare_inputs(input_batch, self.req_states),
vllm/v1/worker/gpu/model_states/whisper.py
+4
-2
@@ -132,7 +132,9 @@ class WhisperModelState(ModelState):
132
num_reqs = input_batch.num_reqs
133
num_tokens = input_batch.num_tokens
134
whisper_attn_metadata = WhisperAttnMetadata(
135
- self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
135
+ self._get_encoder_seq_lens(
136
+ input_batch.req_ids, attn_groups, for_capture, num_reqs
137
+ )
138
)
139
140
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
@@ -166,8 +168,8 @@ class WhisperModelState(ModelState):
168
req_ids: list[str],
169
attn_groups: list[list[AttentionGroup]],
170
for_capture: bool,
171
+ num_reqs: int,
172
) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
170
- num_reqs = len(req_ids)
173
encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
174
if not for_capture:
175
# During normal execution, use actual encoder lengths.