A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Support compile for Transformers multimodal (#23095)
Signed-off-by: raushan <raushan@huggingface.co>
Raushan Turganbay committed
Aug 18, 2025 at 15:35 UTC
0e3bb543f064eb416bca4f6f3013efa3830b12f7
1 file changed
+7
vllm/model_executor/models/transformers.py
+7
@@ -709,6 +709,13 @@ def flatten_and_concat(x: list[torch.Tensor]) -> torch.Tensor:
709
MultiModalProcessor,
710
info=MultiModalProcessingInfo,
711
dummy_inputs=MultiModalDummyInputsBuilder)
712
+@support_torch_compile(
713
+ dynamic_arg_dims={
714
+ "input_ids": 0,
715
+ "positions": -1,
716
+ "intermediate_tensors": 0,
717
+ "inputs_embeds": 0,
718
+ }) # set `positions` to last dim to support Qwen-mrope
719
class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):
720
# Backwards compatibility for prev released models. State dicts back then
721
# had different formats and cannot be loaded with `AutoModel` mapping as is