A high-throughput and memory-efficient inference and serving engine for LLMs
[BugFix] Support async scheduling with prompt embeds for multimodal models (#45673)
Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
Ruinan Ma committed
Jun 15, 2026 at 21:12 UTC
2addbb9cc97e2f75165ab3b81c4287a1dd8a5b0c
2 files changed
+5
-20
vllm/config/vllm.py
-19
@@ -966,15 +966,6 @@ class VllmConfig:
966
"Async scheduling is not compatible with "
967
"disable_padded_drafter_batch=True."
968
)
969
- if (
970
- self.model_config is not None
971
- and self.model_config.enable_prompt_embeds
972
- and self.model_config.is_multimodal_model
973
- ):
974
- raise ValueError(
975
- "Async scheduling is not yet supported with prompt embeds "
976
- "for multimodal models."
977
- )
969
if not executor_supports_async_sched:
970
raise ValueError(
971
f"`{executor_backend}` does not support async scheduling yet."
@@ -1018,16 +1009,6 @@ class VllmConfig:
1009
executor_backend,
1010
)
1011
self.scheduler_config.async_scheduling = False
1021
- elif (
1022
- self.model_config is not None
1023
- and self.model_config.enable_prompt_embeds
1024
- and self.model_config.is_multimodal_model
1025
- ):
1026
- logger.warning_once(
1027
- "Async scheduling is not yet supported with prompt embeds "
1028
- "for multimodal models and will be disabled."
1029
- )
1030
- self.scheduler_config.async_scheduling = False
1012
else:
1013
self.scheduler_config.async_scheduling = True
1014
vllm/v1/worker/gpu_model_runner.py
+5
-1
@@ -1779,13 +1779,17 @@ class GPUModelRunner(
1779
1780
num_common_tokens = len(sample_flattened_indices)
1781
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
1782
+ if self.enable_prompt_embeds:
1783
+ # The multimodal embed path reads is_token_ids.gpu; its .cpu copy is
1784
+ # refreshed every step but the async fast paths below only scatter
1785
+ # input_ids.gpu, so refresh is_token_ids.gpu here too.
1786
+ self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
1787
if num_common_tokens < total_without_spec:
1788
# If not all requests are decodes from the last iteration,
1789
# we need to copy the input_ids_cpu to the GPU first.
1790
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
1791
if self.enable_prompt_embeds:
1792
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
1788
- self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
1793
if num_common_tokens == 0:
1794
# No requests in common with the previous iteration
1795
# So input_ids.cpu will have all the input ids.