A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 13:15 UTC
[LoRA] Make LoRA respect `language_model_only` (#37375)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 18, 2026 at 15:53 UTC
8c31f47c638b87425efc1f3afebf2026336fd061
2 files changed
+14
-3
vllm/lora/model_manager.py
+13
-1
@@ -161,9 +161,9 @@ class LoRAModelManager:
161
device=self.device,
162
lora_config=self.lora_config,
163
)
164
+
165
lm_prefix = self.mm_mapping.language_model[0]
166
self.punica_wrapper_mapping[lm_prefix] = llm_punica_wrapper
166
-
167
if self.lora_config.enable_tower_connector_lora:
168
self.supports_tower_connector_lora = self.supports_mm and hasattr(
169
self.model, "get_num_mm_encoder_tokens"
@@ -171,6 +171,18 @@ class LoRAModelManager:
171
if not self.supports_tower_connector_lora:
172
return
173
174
+ if (
175
+ vllm_config.model_config.multimodal_config
176
+ and vllm_config.model_config.multimodal_config.language_model_only
177
+ ):
178
+ if self.supports_tower_connector_lora:
179
+ logger.warning(
180
+ "Disabling `enable_tower_connector_lora` because the multimodal "
181
+ "model is configured to initialize the language model only."
182
+ )
183
+ self.supports_tower_connector_lora = False
184
+ return
185
+
186
logger.warning(
187
"LoRA for the tower and connector of multimodal models is "
188
"experimental and may contain bugs. Please report any related issues on "
vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py
+1
-2
@@ -10,11 +10,10 @@ from vllm.distributed import (
10
tensor_model_parallel_all_gather,
11
tensor_model_parallel_all_reduce,
12
)
13
+from vllm.lora.ops.triton_ops.utils import supports_pdl
14
from vllm.triton_utils import tl, triton
15
from vllm.utils.torch_utils import direct_register_custom_op
16
16
-from .utils import supports_pdl
17
-
17
18
@triton.jit
19
def _get_lora_id(