e85d1b69cf · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/lora/model_manager.py

+2 -2

@@ -201,8 +201,8 @@ class LoRAModelManager:

201 self.punica_wrapper_mapping[lm_prefix] = llm_punica_wrapper

202

203 # First, determine if the model supports tower connector LoRA.

204 - self.supports_tower_connector_lora = self.supports_mm and hasattr(

205 - self.model, "get_num_mm_encoder_tokens"

204 + self.supports_tower_connector_lora = (

205 + self.supports_mm and self.model.supports_tower_connector_lora

206 )

207

208 # Then, handle the case where the feature is disabled in the config.

vllm/model_executor/models/blip2.py

+2

@@ -525,6 +525,8 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

525 class Blip2ForConditionalGeneration(

526 nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant

527 ):

528 + supports_tower_connector_lora = True

529 +

530 @classmethod

531 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

532 if modality.startswith("image"):

vllm/model_executor/models/dots_ocr.py

+1

@@ -618,6 +618,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA

618 "fc13": ["fc1", "fc3"],

619 }

620 supports_encoder_tp_data = True

621 + supports_tower_connector_lora = True

622

623 @classmethod

624 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

vllm/model_executor/models/gemma3_mm.py

+2

@@ -524,6 +524,8 @@ class Gemma3ForConditionalGeneration(

524 }

525 )

526

527 + supports_tower_connector_lora = True

528 +

529 @classmethod

530 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

531 if modality.startswith("image"):

vllm/model_executor/models/gemma4_mm.py

+1

@@ -978,6 +978,7 @@ class Gemma4ForConditionalGeneration(

978 # window instead of dropping them (which would make image attention

979 # causal-only for images larger than the sliding window).

980 mm_prefix_clamp_sliding_window: bool = True

981 + supports_tower_connector_lora = True

982

983 packed_modules_mapping = {

984 "qkv_proj": [

vllm/model_executor/models/glm4_1v.py

+1

@@ -1755,6 +1755,7 @@ class Glm4vForConditionalGeneration(

1755 )

1756

1757 supports_encoder_tp_data = True

1758 + supports_tower_connector_lora = True

1759

1760 @classmethod

1761 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

vllm/model_executor/models/idefics3.py

+2

@@ -503,6 +503,8 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo

503 ],

504 }

505

506 + supports_tower_connector_lora = True

507 +

508 @classmethod

509 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

510 if modality.startswith("image"):

vllm/model_executor/models/interfaces.py

+6

@@ -167,6 +167,12 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):

167 `multimodal_config.mm_device_do_normalize`.

168 """

169

170 + supports_tower_connector_lora: ClassVar[bool] = False

171 + """

172 + A flag that indicates whether this model supports

173 + `lora_config.enable_tower_connector_lora`.

174 + """

175 +

176 requires_raw_input_tokens: ClassVar[bool] = False

177 """

178 A flag that indicates this model processes input id tokens

vllm/model_executor/models/internvl.py

+1

@@ -553,6 +553,7 @@ class InternVLChatModel(

553 SupportsEncoderCudaGraph,

554 ):

555 supports_encoder_tp_data = True

556 + supports_tower_connector_lora = True

557

558 hf_to_vllm_mapper = WeightsMapper(

559 orig_to_new_prefix=dict.fromkeys(

vllm/model_executor/models/keye.py

+2

@@ -1235,6 +1235,8 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):

1235 }

1236 )

1237

1238 + supports_tower_connector_lora = True

1239 +

1240 @classmethod

1241 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

1242 if modality.startswith("image"):

vllm/model_executor/models/llava.py

+2

@@ -520,6 +520,8 @@ class LlavaForConditionalGeneration(

520 }

521 )

522

523 + supports_tower_connector_lora = True

524 +

525 @classmethod

526 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

527 if modality.startswith("image"):

vllm/model_executor/models/llava_next.py

+1

@@ -231,6 +231,7 @@ class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):

231 class LlavaNextForConditionalGeneration(

232 nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP

233 ):

234 + supports_tower_connector_lora = True

235 packed_modules_mapping = {

236 "qkv_proj": ["q_proj", "k_proj", "v_proj"],

237 "gate_up_proj": ["gate_proj", "up_proj"],

vllm/model_executor/models/llava_next_video.py

+2

@@ -323,6 +323,8 @@ class LlavaNextVideoForConditionalGeneration(

323 }

324 )

325

326 + supports_tower_connector_lora = True

327 +

328 @classmethod

329 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

330 if modality.startswith("video"):

vllm/model_executor/models/mllama4.py

+1

@@ -740,6 +740,7 @@ class Llama4ForConditionalGeneration(

740 }

741

742 supports_encoder_tp_data = True

743 + supports_tower_connector_lora = True

744

745 @classmethod

746 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

vllm/model_executor/models/paligemma.py

+2

@@ -274,6 +274,8 @@ class PaliGemmaForConditionalGeneration(

274 }

275 )

276

277 + supports_tower_connector_lora = True

278 +

279 @classmethod

280 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

281 if modality.startswith("image"):

vllm/model_executor/models/pixtral.py

+2

@@ -309,6 +309,8 @@ class PixtralForConditionalGeneration(

309 "gate_up_proj": ["gate_proj", "up_proj"],

310 }

311

312 + supports_tower_connector_lora = True

313 +

314 @classmethod

315 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

316 if modality.startswith("image"):

vllm/model_executor/models/qwen2_5_vl.py

+1

@@ -1256,6 +1256,7 @@ class Qwen2_5_VLForConditionalGeneration(

1256

1257 supports_encoder_tp_data = True

1258 supports_mm_device_do_normalize = True

1259 + supports_tower_connector_lora = True

1260

1261 def iter_mm_grid_thw(

1262 self, mm_features: list[MultiModalFeatureSpec]

vllm/model_executor/models/qwen2_vl.py

+1

@@ -1194,6 +1194,7 @@ class Qwen2VLForConditionalGeneration(

1194

1195 supports_encoder_tp_data = True

1196 supports_mm_device_do_normalize = True

1197 + supports_tower_connector_lora = True

1198

1199 def iter_mm_grid_thw(

1200 self, mm_features: list[MultiModalFeatureSpec]

vllm/model_executor/models/qwen3_asr.py

+1

@@ -361,6 +361,7 @@ class Qwen3ASRForConditionalGeneration(

361 }

362

363 supported_languages = ISO639_1_SUPPORTED_LANGS

364 + supports_tower_connector_lora = True

365

366 hf_to_vllm_mapper = WeightsMapper(

367 orig_to_new_prefix={

vllm/model_executor/models/qwen3_vl.py

+1

@@ -1753,6 +1753,7 @@ class Qwen3VLForConditionalGeneration(

1753 }

1754

1755 supports_encoder_tp_data = True

1756 + supports_tower_connector_lora = True

1757

1758 supported_video_pruning_methods = ("evs", "vidcom2")

1759

vllm/model_executor/models/ultravox.py

+2

@@ -610,6 +610,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

610 }

611 )

612

613 + supports_tower_connector_lora = True

614 +

615 @classmethod

616 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

617 if modality.startswith("audio"):

[Bugfix][LoRA] Use an explicit capability flag for tower connector LoRA (#53092)