@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1a34c3a5d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 23:00 UTC

[2/N] Initialize MM components in context managers (E-H) (#32641)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 20, 2026 at 16:12 UTC e1a34c3a5d9273431ee182253750d39b546d37a1
12 files changed +163 -191
vllm/model_executor/models/aria.py
-2
@@ -590,8 +590,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
590 def _process_image_input(
591 self, image_input: AriaImagePixelInputs
592 ) -> tuple[torch.Tensor, torch.Tensor]:
593 - assert self.vision_tower is not None
594 -
593 pixel_values = image_input["pixel_values"]
594 pixel_mask = image_input["pixel_mask"]
595
vllm/model_executor/models/aya_vision.py
-1
@@ -382,7 +382,6 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
382 def _process_image_input(
383 self, image_input: AyaVisionImagePixelInputs, **kwargs
384 ) -> list[torch.Tensor]:
385 - assert self.vision_tower is not None
385 pixel_values = image_input["pixel_values"]
386 num_patches = image_input["num_patches"]
387 image_features = self._image_pixels_to_features(
vllm/model_executor/models/cohere2_vision.py
-2
@@ -391,8 +391,6 @@ class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
391 Returns:
392 List of flattened image embeddings, one per image
393 """
394 - assert self.vision_tower is not None, "Vision tower is required"
395 -
394 pixel_values = image_input["pixel_values"]
395 num_patches = image_input["num_patches"]
396
vllm/model_executor/models/ernie45_vl.py
+21 -23
@@ -1303,27 +1303,28 @@ class Ernie4_5_VLMoeForConditionalGeneration(
1303 self.config = config
1304 self.multimodal_config = multimodal_config
1305
1306 - self.vision_model = Ernie4_5_VisionTransformer(
1307 - config.vision_config,
1308 - norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1309 - quant_config=quant_config,
1310 - multimodal_config=multimodal_config,
1311 - prefix=maybe_prefix(prefix, "vision_model"),
1312 - )
1313 -
1314 - self.language_model = Ernie4_5_VLMoeForCausalLM(
1315 - vllm_config=vllm_config,
1316 - prefix=maybe_prefix(prefix, "language_model"),
1317 - )
1306 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1307 + self.vision_model = Ernie4_5_VisionTransformer(
1308 + config.vision_config,
1309 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1310 + quant_config=quant_config,
1311 + multimodal_config=multimodal_config,
1312 + prefix=maybe_prefix(prefix, "vision_model"),
1313 + )
1314 + self.resampler_model = VariableResolutionResamplerModel(
1315 + self.config.pixel_hidden_size,
1316 + self.config.hidden_size,
1317 + self.config.spatial_conv_size,
1318 + self.config.temporal_conv_size,
1319 + config=self.config,
1320 + prefix=maybe_prefix(prefix, "resampler_model"),
1321 + )
1322
1319 - self.resampler_model = VariableResolutionResamplerModel(
1320 - self.config.pixel_hidden_size,
1321 - self.config.hidden_size,
1322 - self.config.spatial_conv_size,
1323 - self.config.temporal_conv_size,
1324 - config=self.config,
1325 - prefix=maybe_prefix(prefix, "resampler_model"),
1326 - )
1323 + with self._mark_language_model(vllm_config):
1324 + self.language_model = Ernie4_5_VLMoeForCausalLM(
1325 + vllm_config=vllm_config,
1326 + prefix=maybe_prefix(prefix, "language_model"),
1327 + )
1328
1329 self.visual_token_mask = None
1330 self.make_empty_intermediate_tensors = (
@@ -1522,9 +1523,6 @@ class Ernie4_5_VLMoeForConditionalGeneration(
1523 mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1524 return llm_positions, mrope_position_delta
1525
1525 - def get_language_model(self) -> torch.nn.Module:
1526 - return self.language_model
1527 -
1526 def _parse_and_validate_image_input(
1527 self, **kwargs: object
1528 ) -> Ernie4_5_VLImageInputs | None:
vllm/model_executor/models/fuyu.py
+15 -18
@@ -287,16 +287,20 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
287 self.image_token_id = _IMAGE_TOKEN_ID
288 self.image_feature_size = config.patch_size**2 * config.num_channels
289
290 - self.vision_embed_tokens = ColumnParallelLinear(
291 - self.image_feature_size,
292 - config.hidden_size,
293 - quant_config=quant_config,
294 - gather_output=True,
295 - )
296 - self.language_model = PersimmonForCausalLM(
297 - vllm_config=vllm_config.with_hf_config(config.text_config),
298 - prefix=maybe_prefix(prefix, "language_model"),
299 - )
290 + with self._mark_tower_model(vllm_config, "image"):
291 + self.vision_embed_tokens = ColumnParallelLinear(
292 + self.image_feature_size,
293 + config.hidden_size,
294 + quant_config=quant_config,
295 + gather_output=True,
296 + )
297 +
298 + with self._mark_language_model(vllm_config):
299 + self.language_model = PersimmonForCausalLM(
300 + vllm_config=vllm_config.with_hf_config(config.text_config),
301 + prefix=maybe_prefix(prefix, "language_model"),
302 + )
303 +
304 self.make_empty_intermediate_tensors = (
305 self.language_model.make_empty_intermediate_tensors
306 )
@@ -323,14 +327,10 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
327 image_patches_flat = image_input["image_patches_flat"]
328 patches_per_image = image_input["patches_per_image"]
329
326 - assert self.vision_embed_tokens is not None
330 vision_embeddings_flat, _ = self.vision_embed_tokens(image_patches_flat)
331
332 return vision_embeddings_flat.split(patches_per_image.tolist(), dim=0)
333
331 - def get_language_model(self) -> torch.nn.Module:
332 - return self.language_model
333 -
334 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
335 image_input = self._parse_and_validate_image_input(**kwargs)
336 if image_input is None:
@@ -361,10 +361,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
361 self,
362 hidden_states: torch.Tensor,
363 ) -> torch.Tensor | None:
364 - logits = self.language_model.logits_processor(
365 - self.language_model.lm_head, hidden_states
366 - )
367 - return logits
364 + return self.language_model.compute_logits(hidden_states)
365
366 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
367 loader = AutoWeightsLoader(self)
vllm/model_executor/models/gemma3_mm.py
+20 -23
@@ -522,25 +522,27 @@ class Gemma3ForConditionalGeneration(
522 self.quant_config = quant_config
523 self.multimodal_config = multimodal_config
524
525 - self.vision_tower = SiglipVisionModel(
526 - config.vision_config,
527 - quant_config,
528 - prefix=maybe_prefix(prefix, "vision_tower"),
529 - )
530 - self.multi_modal_projector = Gemma3MultiModalProjector(config)
531 -
532 - self.language_model = init_vllm_registered_model(
533 - vllm_config=vllm_config,
534 - hf_config=config.text_config,
535 - prefix=maybe_prefix(prefix, "language_model"),
536 - architectures=["Gemma3ForCausalLM"],
537 - )
538 - logit_scale = getattr(config, "logit_scale", 1.0)
525 + with self._mark_tower_model(vllm_config, "image"):
526 + self.vision_tower = SiglipVisionModel(
527 + config.vision_config,
528 + quant_config,
529 + prefix=maybe_prefix(prefix, "vision_tower"),
530 + )
531 + self.multi_modal_projector = Gemma3MultiModalProjector(config)
532 +
533 + with self._mark_language_model(vllm_config):
534 + self.language_model = init_vllm_registered_model(
535 + vllm_config=vllm_config,
536 + hf_config=config.text_config,
537 + prefix=maybe_prefix(prefix, "language_model"),
538 + architectures=["Gemma3ForCausalLM"],
539 + )
540
540 - if hasattr(self.language_model, "logits_processor"):
541 - # The logits processor can be unset if we're using
542 - # automatic conversion to pooling model.
543 - self.language_model.logits_processor.scale *= logit_scale
541 + logit_scale = getattr(config, "logit_scale", 1.0)
542 + if hasattr(self.language_model, "logits_processor"):
543 + # The logits processor can be unset if we're using
544 + # automatic conversion to pooling model.
545 + self.language_model.logits_processor.scale *= logit_scale
546
547 self.make_empty_intermediate_tensors = (
548 self.language_model.make_empty_intermediate_tensors
@@ -579,8 +581,6 @@ class Gemma3ForConditionalGeneration(
581 self,
582 image_input: Gemma3ImageInputs,
583 ) -> list[torch.Tensor]:
582 - assert self.vision_tower is not None
583 -
584 pixel_values = image_input["pixel_values"]
585 num_patches = image_input["num_patches"]
586
@@ -592,9 +592,6 @@ class Gemma3ForConditionalGeneration(
592
593 return [e.flatten(0, 1) for e in image_embeds.split(num_patches.tolist())]
594
595 - def get_language_model(self) -> torch.nn.Module:
596 - return self.language_model
597 -
595 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
596 image_input = self._parse_and_validate_image_input(**kwargs)
597 if image_input is None:
vllm/model_executor/models/gemma3n_mm.py
+28 -30
@@ -503,31 +503,35 @@ class Gemma3nForConditionalGeneration(
503 self.multimodal_config = multimodal_config
504 self.vocab_size = config.text_config.vocab_size
505
506 - self.vision_tower = AutoModel.from_config(config=config.vision_config)
507 - self.audio_tower = AutoModel.from_config(config=config.audio_config)
508 - self.embed_vision = Gemma3nMultimodalEmbedder(
509 - config.vision_config, config.text_config
510 - )
511 - self.embed_audio = Gemma3nMultimodalEmbedder(
512 - config.audio_config, config.text_config
513 - )
506 + with self._mark_tower_model(vllm_config, "image"):
507 + self.vision_tower = AutoModel.from_config(config=config.vision_config)
508 + self.embed_vision = Gemma3nMultimodalEmbedder(
509 + config.vision_config, config.text_config
510 + )
511
515 - self.language_model: nn.Module = init_vllm_registered_model(
516 - vllm_config=vllm_config,
517 - hf_config=config.text_config,
518 - prefix=maybe_prefix(prefix, "language_model"),
519 - architectures=["Gemma3nForCausalLM"],
520 - )
521 - self.language_model = cast(Gemma3nForCausalLM, self.language_model)
522 - # NOTE (NickLucche) In order to be compatible with cudagraph, the
523 - # buffer needs to be consistent, so we pre-allocate here.
524 - self.per_layer_embeddings = torch.zeros(
525 - vllm_config.scheduler_config.max_num_batched_tokens,
526 - self.config.text_config.num_hidden_layers,
527 - self.config.text_config.hidden_size_per_layer_input,
528 - device=self.language_model.model.embed_tokens.weight.device,
529 - dtype=self.language_model.model.embed_tokens.weight.dtype,
530 - )
512 + with self._mark_tower_model(vllm_config, "audio"):
513 + self.audio_tower = AutoModel.from_config(config=config.audio_config)
514 + self.embed_audio = Gemma3nMultimodalEmbedder(
515 + config.audio_config, config.text_config
516 + )
517 +
518 + with self._mark_language_model(vllm_config):
519 + self.language_model: Gemma3nForCausalLM = init_vllm_registered_model(
520 + vllm_config=vllm_config,
521 + hf_config=config.text_config,
522 + prefix=maybe_prefix(prefix, "language_model"),
523 + architectures=["Gemma3nForCausalLM"],
524 + )
525 +
526 + # NOTE (NickLucche) In order to be compatible with cudagraph, the
527 + # buffer needs to be consistent, so we pre-allocate here.
528 + self.per_layer_embeddings = torch.zeros(
529 + vllm_config.scheduler_config.max_num_batched_tokens,
530 + self.config.text_config.num_hidden_layers,
531 + self.config.text_config.hidden_size_per_layer_input,
532 + device=self.language_model.model.embed_tokens.weight.device,
533 + dtype=self.language_model.model.embed_tokens.weight.dtype,
534 + )
535
536 def _parse_and_validate_image_input(
537 self, **kwargs: object
@@ -583,8 +587,6 @@ class Gemma3nForConditionalGeneration(
587 self,
588 image_input: Gemma3nImageInputs,
589 ) -> list[torch.Tensor]:
586 - assert self.vision_tower is not None
587 -
590 pixel_values = image_input["pixel_values"]
591 vision_outputs = self.vision_tower(
592 pixel_values=pixel_values, do_pooling=False, return_dict=True
@@ -609,7 +611,6 @@ class Gemma3nForConditionalGeneration(
611 self,
612 audio_input: Gemma3nAudioInputs,
613 ) -> list[torch.Tensor]:
612 - assert self.audio_tower is not None
614 # Run on padded features to enable batching
615 input_features = audio_input["input_features_padded"].squeeze(1)
616 input_features_mask = audio_input["input_features_mask"].squeeze(1)
@@ -651,9 +652,6 @@ class Gemma3nForConditionalGeneration(
652 # Return a list of embeddings instead of a batched tensor
653 return audio_features.unbind(0)
654
654 - def get_language_model(self) -> torch.nn.Module:
655 - return self.language_model
656 -
655 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
656 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
657 if mm_input_by_modality is None:
vllm/model_executor/models/glm4_1v.py
+15 -16
@@ -1434,13 +1434,14 @@ class Glm4vForConditionalGeneration(
1434 self.multimodal_config = multimodal_config
1435 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1436
1437 - self.visual = Glm4vVisionTransformer(
1438 - config.vision_config,
1439 - norm_eps=getattr(config, "rms_norm_eps", 1e-5),
1440 - quant_config=quant_config,
1441 - multimodal_config=multimodal_config,
1442 - prefix=maybe_prefix(prefix, "visual"),
1443 - )
1437 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1438 + self.visual = Glm4vVisionTransformer(
1439 + config.vision_config,
1440 + norm_eps=getattr(config, "rms_norm_eps", 1e-5),
1441 + quant_config=quant_config,
1442 + multimodal_config=multimodal_config,
1443 + prefix=maybe_prefix(prefix, "visual"),
1444 + )
1445
1446 if config.model_type == "glm4v":
1447 architectures = ["Glm4ForCausalLM"]
@@ -1449,12 +1450,13 @@ class Glm4vForConditionalGeneration(
1450 else:
1451 architectures = None
1452
1452 - self.language_model = init_vllm_registered_model(
1453 - vllm_config=vllm_config,
1454 - hf_config=config.text_config,
1455 - prefix=maybe_prefix(prefix, "language_model"),
1456 - architectures=architectures,
1457 - )
1453 + with self._mark_language_model(vllm_config):
1454 + self.language_model = init_vllm_registered_model(
1455 + vllm_config=vllm_config,
1456 + hf_config=config.text_config,
1457 + prefix=maybe_prefix(prefix, "language_model"),
1458 + architectures=architectures,
1459 + )
1460
1461 self.make_empty_intermediate_tensors = (
1462 self.language_model.make_empty_intermediate_tensors
@@ -1578,9 +1580,6 @@ class Glm4vForConditionalGeneration(
1580 )
1581 return mm_input_by_modality
1582
1581 - def get_language_model(self) -> torch.nn.Module:
1582 - return self.language_model
1583 -
1583 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
1584 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
1585 if not mm_input_by_modality:
vllm/model_executor/models/glmasr.py
+19 -21
@@ -944,26 +944,27 @@ class GlmAsrForConditionalGeneration(
944 multimodal_config = vllm_config.model_config.multimodal_config
945 self.config = config
946 self.multimodal_config = multimodal_config
947 -
948 - # Use optimized vLLM native encoder
949 - self.audio_tower = GlmAsrEncoder(
950 - config.audio_config,
951 - quant_config=quant_config,
952 - prefix=maybe_prefix(prefix, "audio_tower"),
953 - )
954 - self.multi_modal_projector = GlmAsrMultiModalProjector(
955 - config,
956 - quant_config=quant_config,
957 - prefix=maybe_prefix(prefix, "multi_modal_projector"),
958 - )
947 self.quant_config = quant_config
948
961 - self.language_model = init_vllm_registered_model(
962 - vllm_config=vllm_config,
963 - hf_config=config.text_config,
964 - prefix=maybe_prefix(prefix, "language_model"),
965 - architectures=["LlamaForCausalLM"],
966 - )
949 + with self._mark_tower_model(vllm_config, "audio"):
950 + self.audio_tower = GlmAsrEncoder(
951 + config.audio_config,
952 + quant_config=quant_config,
953 + prefix=maybe_prefix(prefix, "audio_tower"),
954 + )
955 + self.multi_modal_projector = GlmAsrMultiModalProjector(
956 + config,
957 + quant_config=quant_config,
958 + prefix=maybe_prefix(prefix, "multi_modal_projector"),
959 + )
960 +
961 + with self._mark_language_model(vllm_config):
962 + self.language_model = init_vllm_registered_model(
963 + vllm_config=vllm_config,
964 + hf_config=config.text_config,
965 + prefix=maybe_prefix(prefix, "language_model"),
966 + architectures=["LlamaForCausalLM"],
967 + )
968
969 self.make_empty_intermediate_tensors = (
970 self.language_model.make_empty_intermediate_tensors
@@ -1063,9 +1064,6 @@ class GlmAsrForConditionalGeneration(
1064 )
1065 return _group_audio_embeddings(chunk_embeddings, chunk_counts)
1066
1066 - def get_language_model(self) -> torch.nn.Module:
1067 - return self.language_model
1068 -
1067 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1068 audio_input = self._parse_and_validate_audio_input(**kwargs)
1069 if audio_input is None:
vllm/model_executor/models/granite_speech.py
+21 -22
@@ -597,27 +597,29 @@ class GraniteSpeechForConditionalGeneration(
597 self.quant_config = quant_config
598 self.cache_config = cache_config
599
600 - # The language model is typically a Granite LLM
601 - self.language_model = init_vllm_registered_model(
602 - vllm_config=vllm_config,
603 - hf_config=config.text_config,
604 - prefix=maybe_prefix(prefix, "language_model"),
605 - )
600 + with self._mark_language_model(vllm_config):
601 + # The language model is typically a Granite LLM
602 + self.language_model = init_vllm_registered_model(
603 + vllm_config=vllm_config,
604 + hf_config=config.text_config,
605 + prefix=maybe_prefix(prefix, "language_model"),
606 + )
607
607 - # Conformer encoder
608 - self.encoder = GraniteSpeechCTCEncoder(
609 - config=config.encoder_config,
610 - quant_config=quant_config,
611 - prefix=f"{prefix}.encoder",
612 - )
608 + with self._mark_tower_model(vllm_config, "audio"):
609 + # Conformer encoder
610 + self.encoder = GraniteSpeechCTCEncoder(
611 + config=config.encoder_config,
612 + quant_config=quant_config,
613 + prefix=f"{prefix}.encoder",
614 + )
615
614 - # Blip2 QFormer
615 - self.projector = GraniteSpeechEncoderProjector(
616 - config=config,
617 - quant_config=quant_config,
618 - cache_config=cache_config,
619 - prefix=f"{prefix}.projector",
620 - )
616 + # Blip2 QFormer
617 + self.projector = GraniteSpeechEncoderProjector(
618 + config=config,
619 + quant_config=quant_config,
620 + cache_config=cache_config,
621 + prefix=f"{prefix}.projector",
622 + )
623
624 self.make_empty_intermediate_tensors = (
625 self.language_model.make_empty_intermediate_tensors
@@ -770,9 +772,6 @@ class GraniteSpeechForConditionalGeneration(
772 # Split variable length features into a tuple
773 return torch.split(masked_embeds, audio_input["audio_embed_sizes"])
774
773 - def get_language_model(self) -> torch.nn.Module:
774 - return self.language_model
775 -
775 def embed_multimodal(
776 self,
777 **kwargs: object,
vllm/model_executor/models/hunyuan_vision.py
+11 -15
@@ -877,7 +877,7 @@ class HunYuanVLForConditionalGeneration(
877 self.config = config
878 self.multimodal_config = multimodal_config
879
880 - if multimodal_config.get_limit_per_prompt("image"):
880 + with self._mark_tower_model(vllm_config, {"image"}):
881 attn_backend_override = (
882 multimodal_config.mm_encoder_attn_backend
883 if multimodal_config is not None
@@ -890,17 +890,16 @@ class HunYuanVLForConditionalGeneration(
890 multimodal_config=multimodal_config,
891 attn_backend_override=attn_backend_override,
892 )
893 - else:
894 - self.visual = None
895 -
896 - self.language_model = init_vllm_registered_model(
897 - vllm_config=vllm_config,
898 - prefix=maybe_prefix(prefix, "language_model.model"),
899 - architectures=[
900 - "HunYuanDenseV1ForCausalLM",
901 - "HunYuanMoEV1ForCausalLM",
902 - ],
903 - )
893 +
894 + with self._mark_language_model(vllm_config):
895 + self.language_model = init_vllm_registered_model(
896 + vllm_config=vllm_config,
897 + prefix=maybe_prefix(prefix, "language_model.model"),
898 + architectures=[
899 + "HunYuanDenseV1ForCausalLM",
900 + "HunYuanMoEV1ForCausalLM",
901 + ],
902 + )
903
904 self.make_empty_intermediate_tensors = (
905 self.language_model.make_empty_intermediate_tensors
@@ -970,9 +969,6 @@ class HunYuanVLForConditionalGeneration(
969 )
970 return mm_input_by_modality
971
973 - def get_language_model(self) -> torch.nn.Module:
974 - return self.language_model
975 -
972 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
973 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
974 if not mm_input_by_modality:
vllm/model_executor/models/hyperclovax_vision.py
+13 -18
@@ -15,7 +15,6 @@ from einops import rearrange
15 from timm.layers import LayerNorm, LayerNorm2d
16 from timm.models.regnet import RegStage
17 from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
18 -from transformers.modeling_utils import no_init_weights
18
19 from vllm.config import VllmConfig
20 from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig
@@ -625,8 +624,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
624 config, vision_config
625 )
626
628 - # init models & parameters
629 - with no_init_weights(): # weight will be loaded in from_pretrained
627 + with self._mark_tower_model(vllm_config, {"image", "video"}):
628 self.vision_model = init_vision_tower_for_hcxvision(
629 vision_config,
630 quant_config=quant_config,
@@ -635,20 +633,20 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
633 require_post_norm=False,
634 prefix=maybe_prefix(prefix, "vision_model"),
635 )
638 - self.mm_projector = self._init_mm_projector(config, text_config, vision_config)
636 + self.mm_projector = self._init_mm_projector(
637 + config, text_config, vision_config
638 + )
639
640 - self.lm_head_vocab_size = getattr(
641 - text_config, "padded_vocab_size", text_config.vocab_size
642 - )
643 - self.language_model = init_vllm_registered_model(
644 - vllm_config=vllm_config,
645 - hf_config=text_config,
646 - prefix=maybe_prefix(prefix, "language_model"),
647 - )
640 + if config.anyres:
641 + self.image_newline = nn.Parameter(
642 + torch.empty(text_config.hidden_size, dtype=self.dtype)
643 + )
644
649 - if config.anyres:
650 - self.image_newline = nn.Parameter(
651 - torch.empty(text_config.hidden_size, dtype=self.dtype)
645 + with self._mark_language_model(vllm_config):
646 + self.language_model = init_vllm_registered_model(
647 + vllm_config=vllm_config,
648 + hf_config=text_config,
649 + prefix=maybe_prefix(prefix, "language_model"),
650 )
651
652 self.config = config
@@ -726,9 +724,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
724
725 return modalities
726
729 - def get_language_model(self) -> torch.nn.Module:
730 - return self.language_model
731 -
727 def embed_multimodal(
728 self,
729 **kwargs: object,