A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[Model] Remove unnecessary `get_language_model` (#37545)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 19, 2026 at 20:00 UTC
7a6ebcbfcf2e74876d8493903d444625cd221e7e
7 files changed
+93
-95
vllm/model_executor/models/cohere_asr.py
+21
-8
@@ -1704,6 +1704,12 @@ class ConformerEncoder(nn.Module):
1704
# ----- Encoder END -----
1705
1706
1707
+# This subclass is specific to vLLM in order for
1708
+# `_mark_composite_model` to target this module
1709
+class CohereASRProjector(nn.Linear):
1710
+ pass
1711
+
1712
+
1713
class CohereASRModel(nn.Module):
1714
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1715
super().__init__()
@@ -1714,7 +1720,7 @@ class CohereASRModel(nn.Module):
1720
)
1721
1722
if self.encoder.d_model != self.decoder.hidden_size:
1717
- self.encoder_decoder_proj = torch.nn.Linear(
1723
+ self.encoder_decoder_proj = CohereASRProjector(
1724
self.encoder.d_model, self.decoder.hidden_size
1725
)
1726
@@ -2096,18 +2102,25 @@ class CohereASRForConditionalGeneration(
2102
self.config = config
2103
self.dtype = vllm_config.model_config.dtype
2104
2099
- self.model = CohereASRModel(vllm_config=vllm_config, prefix=prefix)
2100
- lm_head_config = config.head
2101
- self.unpadded_vocab_size = lm_head_config["num_classes"]
2105
+ with self._mark_composite_model(
2106
+ vllm_config,
2107
+ language_targets=CohereASRDecoder,
2108
+ tower_targets={"audio": (ConformerEncoder, CohereASRProjector)},
2109
+ ):
2110
+ self.model = CohereASRModel(vllm_config=vllm_config, prefix=prefix)
2111
+
2112
+ head_config = config.head
2113
+
2114
self.proj_out = ParallelLMHead(
2103
- lm_head_config["num_classes"],
2104
- lm_head_config["hidden_size"],
2115
+ head_config["num_classes"],
2116
+ head_config["hidden_size"],
2117
quant_config=quant_config,
2118
bias=True,
2119
) # NOTE: bias is True
2108
- logit_scale = getattr(lm_head_config, "logit_scale", 1.0)
2120
+
2121
+ logit_scale = getattr(head_config, "logit_scale", 1.0)
2122
self.logits_processor = LogitsProcessor(
2110
- self.unpadded_vocab_size, lm_head_config["num_classes"], logit_scale
2123
+ head_config["num_classes"], scale=logit_scale
2124
)
2125
2126
def forward(
vllm/model_executor/models/ernie45_vl.py
-1
@@ -1373,7 +1373,6 @@ class Ernie4_5_VLMoeForConditionalGeneration(
1373
self,
1374
hidden_states: torch.Tensor,
1375
) -> torch.Tensor | None:
1376
- """compute logits"""
1376
return self.language_model.compute_logits(hidden_states)
1377
1378
def _vision_forward(
vllm/model_executor/models/fireredasr2.py
+10
-5
@@ -754,12 +754,17 @@ class FireRedASR2ForConditionalGeneration(
754
self.config = config
755
self.dtype = vllm_config.model_config.dtype
756
757
- self.model = FireRedASR2Model(
758
- vllm_config=vllm_config,
759
- prefix=maybe_prefix(prefix, "model"),
760
- )
761
- logit_scale = getattr(config, "logit_scale", 1.0)
757
+ with self._mark_composite_model(
758
+ vllm_config,
759
+ language_targets=Qwen2ForCausalLM,
760
+ tower_targets={"audio": (FireRedASR2Encoder, FireRedASR2Adapter)},
761
+ ):
762
+ self.model = FireRedASR2Model(
763
+ vllm_config=vllm_config,
764
+ prefix=maybe_prefix(prefix, "model"),
765
+ )
766
767
+ logit_scale = getattr(config, "logit_scale", 1.0)
768
self.logits_processor = LogitsProcessor(config.vocab_size, scale=logit_scale)
769
770
def forward(
vllm/model_executor/models/hyperclovax_vision_v2.py
+14
-23
@@ -470,15 +470,6 @@ class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
470
self.vision_config = vision_config
471
self.text_config = text_config
472
self.vllm_config = vllm_config
473
- self.dtype = vllm_config.model_config.dtype
474
-
475
- # Initialize Qwen2.5 Vision Transformer
476
- self.visual = Qwen2_5_VisionTransformer(
477
- vision_config=vision_config,
478
- norm_eps=getattr(config, "rms_norm_eps", 1e-6),
479
- quant_config=quant_config,
480
- prefix=maybe_prefix(prefix, "visual"),
481
- )
473
474
# Linear projector (vision_hidden_size -> text_hidden_size)
475
# For V2 model: mm_projector_type is "linear"
@@ -492,18 +483,21 @@ class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
483
else:
484
out_hidden = vision_hidden_size
485
495
- # Always create Linear projector since HF checkpoint has mm_projector weights
496
- self.mm_projector = nn.Linear(out_hidden, text_hidden_size)
486
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
487
+ self.visual = Qwen2_5_VisionTransformer(
488
+ vision_config=vision_config,
489
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
490
+ quant_config=quant_config,
491
+ prefix=maybe_prefix(prefix, "visual"),
492
+ )
493
+ self.mm_projector = nn.Linear(out_hidden, text_hidden_size)
494
498
- # Language model
499
- self.lm_head_vocab_size = getattr(
500
- text_config, "padded_vocab_size", text_config.vocab_size
501
- )
502
- self.language_model = init_vllm_registered_model(
503
- vllm_config=vllm_config,
504
- hf_config=text_config,
505
- prefix=maybe_prefix(prefix, "language_model"),
506
- )
495
+ with self._mark_language_model(vllm_config):
496
+ self.language_model = init_vllm_registered_model(
497
+ vllm_config=vllm_config,
498
+ hf_config=text_config,
499
+ prefix=maybe_prefix(prefix, "language_model"),
500
+ )
501
502
self.make_empty_intermediate_tensors = (
503
self.language_model.make_empty_intermediate_tensors
@@ -633,9 +627,6 @@ class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
627
628
return modalities
629
636
- def get_language_model(self) -> torch.nn.Module:
637
- return self.language_model
638
-
630
def embed_multimodal(
631
self,
632
**kwargs: object,
vllm/model_executor/models/interns1_pro.py
+7
-8
@@ -576,20 +576,19 @@ class InternS1ProForConditionalGeneration(
576
multimodal_config.is_multimodal_pruning_enabled()
577
)
578
579
- if not multimodal_config.get_limit_per_prompt(
580
- "image"
581
- ) and not multimodal_config.get_limit_per_prompt("video"):
582
- self.visual = None
583
- else:
579
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
580
self.visual = Qwen3_VisionTransformer(
581
config.vision_config,
582
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
583
prefix=maybe_prefix(prefix, "visual"),
584
)
585
590
- self.language_model = InternS1ProMoeLLMForCausalLM(
591
- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
592
- )
586
+ with self._mark_language_model(vllm_config):
587
+ self.language_model = InternS1ProMoeLLMForCausalLM(
588
+ vllm_config=vllm_config,
589
+ prefix=maybe_prefix(prefix, "language_model"),
590
+ )
591
+
592
# Whether to include the gate_up_proj mapping is determined by
593
# the language model.
594
self.packed_modules_mapping = (
vllm/model_executor/models/kimi_audio.py
+18
-28
@@ -15,7 +15,6 @@ from transformers import WhisperConfig as HFWhisperConfig
15
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
16
from vllm.config.multimodal import BaseDummyOptions
17
from vllm.inputs.data import PromptType, TokensPrompt
18
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.model_loader import DefaultModelLoader
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.model_executor.models.interfaces import (
@@ -54,7 +53,6 @@ from vllm.tokenizers import cached_get_tokenizer
53
from vllm.tokenizers.kimi_audio import KimiAudioTokenizer
54
from vllm.transformers_utils.processor import cached_feature_extractor_from_config
55
from vllm.transformers_utils.processors.kimi_audio import KimiAudioProcessor
57
-from vllm.v1.sample.metadata import SamplingMetadata
56
57
# Kimi-Audio constants
58
KIMIA_WHISPER_SUBFOLDER = "whisper-large-v3"
@@ -431,28 +429,24 @@ class KimiAudioForConditionalGeneration(
429
)
430
]
431
434
- self.audio_tower = KimiAudioWhisperEncoder(
435
- vllm_config=vllm_config,
436
- prefix=maybe_prefix(prefix, "audio_tower"),
437
- )
438
-
439
- self.multi_modal_projector = KimiAudioMultiModalProjector(
440
- whisper_dim=getattr(self.config, "kimia_adaptor_input_dim", 5120),
441
- llm_dim=self.config.hidden_size,
442
- prefix=maybe_prefix(prefix, "multi_modal_projector"),
443
- )
444
-
445
- self.language_model = init_vllm_registered_model(
446
- vllm_config=vllm_config.with_hf_config(
447
- self.config, architectures=["Qwen2ForCausalLM"]
448
- ),
449
- prefix=maybe_prefix(prefix, "language_model"),
450
- )
432
+ with self._mark_tower_model(vllm_config, "audio"):
433
+ self.audio_tower = KimiAudioWhisperEncoder(
434
+ vllm_config=vllm_config,
435
+ prefix=maybe_prefix(prefix, "audio_tower"),
436
+ )
437
+ self.multi_modal_projector = KimiAudioMultiModalProjector(
438
+ whisper_dim=getattr(self.config, "kimia_adaptor_input_dim", 5120),
439
+ llm_dim=self.config.hidden_size,
440
+ prefix=maybe_prefix(prefix, "multi_modal_projector"),
441
+ )
442
452
- self.logits_processor = LogitsProcessor(
453
- self.config.vocab_size,
454
- self.config.vocab_size,
455
- )
443
+ with self._mark_language_model(vllm_config):
444
+ self.language_model = init_vllm_registered_model(
445
+ vllm_config=vllm_config.with_hf_config(
446
+ self.config, architectures=["Qwen2ForCausalLM"]
447
+ ),
448
+ prefix=maybe_prefix(prefix, "language_model"),
449
+ )
450
451
self.make_empty_intermediate_tensors = (
452
self.language_model.make_empty_intermediate_tensors
@@ -595,12 +589,8 @@ class KimiAudioForConditionalGeneration(
589
def compute_logits(
590
self,
591
hidden_states: torch.Tensor,
598
- sampling_metadata: SamplingMetadata | None = None,
592
) -> torch.Tensor | None:
600
- logits = self.logits_processor(
601
- self.language_model.lm_head, hidden_states, sampling_metadata
602
- )
603
- return logits
593
+ return self.language_model.compute_logits(hidden_states)
594
595
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
596
"""Load weights, skipping MIMO layers (TTS-only) for ASR."""
vllm/model_executor/models/lightonocr.py
+23
-22
@@ -163,29 +163,30 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
163
self.config = config
164
self.multimodal_config = multimodal_config
165
166
- self.vision_tower = init_vision_tower_for_llava(
167
- config,
168
- quant_config=quant_config,
169
- require_post_norm=False,
170
- prefix=maybe_prefix(prefix, "vision_tower"),
171
- )
172
-
173
- self.multi_modal_projector = Mistral3MultiModalProjector(
174
- vision_hidden_size=config.vision_config.hidden_size,
175
- text_hidden_size=config.text_config.hidden_size,
176
- projector_hidden_act=config.projector_hidden_act,
177
- spatial_merge_size=config.spatial_merge_size,
178
- patch_size=config.vision_config.patch_size,
179
- multimodal_projector_bias=config.multimodal_projector_bias,
180
- quant_config=quant_config,
181
- prefix=maybe_prefix(prefix, "multi_modal_projector"),
182
- )
166
+ with self._mark_tower_model(vllm_config, "image"):
167
+ self.vision_tower = init_vision_tower_for_llava(
168
+ config,
169
+ quant_config=quant_config,
170
+ require_post_norm=False,
171
+ prefix=maybe_prefix(prefix, "vision_tower"),
172
+ )
173
+ self.multi_modal_projector = Mistral3MultiModalProjector(
174
+ vision_hidden_size=config.vision_config.hidden_size,
175
+ text_hidden_size=config.text_config.hidden_size,
176
+ projector_hidden_act=config.projector_hidden_act,
177
+ spatial_merge_size=config.spatial_merge_size,
178
+ patch_size=config.vision_config.patch_size,
179
+ multimodal_projector_bias=config.multimodal_projector_bias,
180
+ quant_config=quant_config,
181
+ prefix=maybe_prefix(prefix, "multi_modal_projector"),
182
+ )
183
184
- self.language_model = init_vllm_registered_model(
185
- vllm_config=vllm_config,
186
- hf_config=config.text_config,
187
- prefix=maybe_prefix(prefix, "language_model"),
188
- )
184
+ with self._mark_language_model(vllm_config):
185
+ self.language_model = init_vllm_registered_model(
186
+ vllm_config=vllm_config,
187
+ hf_config=config.text_config,
188
+ prefix=maybe_prefix(prefix, "language_model"),
189
+ )
190
191
self.make_empty_intermediate_tensors = (
192
self.language_model.make_empty_intermediate_tensors