A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Use Transformers helper `get_text_config()` instead of checking for `text_config` (#17105)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 25, 2025 at 16:47 UTC
423e9f1cbe90c206f2dd9d18178546b1058b3f67
7 files changed
+30
-46
benchmarks/kernels/benchmark_moe.py
+2
-3
@@ -553,9 +553,8 @@ def main(args: argparse.Namespace):
553
intermediate_size = config.moe_intermediate_size
554
shard_intermediate_size = 2 * intermediate_size // args.tp_size
555
else:
556
- if not hasattr(config, "hidden_size"):
557
- # Support for llama4
558
- config = config.text_config
556
+ # Support for llama4
557
+ config = config.get_text_config()
558
# Default: Mixtral.
559
E = config.num_local_experts
560
topk = config.num_experts_per_tok
tests/models/test_initialization.py
+1
-4
@@ -24,10 +24,7 @@ def test_can_initialize(model_arch):
24
def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
25
hf_config.update(model_info.hf_overrides)
26
27
- if hasattr(hf_config, "text_config"):
28
- text_config: PretrainedConfig = hf_config.text_config
29
- else:
30
- text_config = hf_config
27
+ text_config = hf_config.get_text_config()
28
29
text_config.update({
30
"num_layers": 1,
vllm/config.py
+2
-4
@@ -2841,12 +2841,10 @@ def _get_and_verify_dtype(
2841
) -> torch.dtype:
2842
# NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
2843
# because config.torch_dtype can be None.
2844
- config_dtype = getattr(config, "torch_dtype", None)
2844
+ config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
2845
2846
- # Fallbacks for multi-modal models if the root config
2846
+ # Fallback for multi-modal models if the root config
2847
# does not define torch_dtype
2848
- if config_dtype is None and hasattr(config, "text_config"):
2849
- config_dtype = getattr(config.text_config, "torch_dtype", None)
2848
if config_dtype is None and hasattr(config, "vision_config"):
2849
config_dtype = getattr(config.vision_config, "torch_dtype", None)
2850
vllm/transformers_utils/config.py
+12
-9
@@ -760,19 +760,22 @@ def get_hf_text_config(config: PretrainedConfig):
760
"""Get the "sub" config relevant to llm for multi modal models.
761
No op for pure text models.
762
"""
763
- if hasattr(config, "text_config"):
764
- # The code operates under the assumption that text_config should have
765
- # `num_attention_heads` (among others). Assert here to fail early
766
- # if transformers config doesn't align with this assumption.
767
- assert hasattr(config.text_config, "num_attention_heads")
768
- return config.text_config
769
- elif hasattr(config, "thinker_config"):
763
+ # This block should be unnecessary after https://github.com/huggingface/transformers/pull/37517
764
+ if hasattr(config, "thinker_config"):
765
# TODO(suyang.fy): Refactor code.
766
# For Qwen2.5-Omni, change hf_text_config to
767
# thinker_config.text_config.
768
return config.thinker_config.text_config
774
- else:
775
- return config
769
+
770
+ text_config = config.get_text_config()
771
+
772
+ if text_config is not config:
773
+ # The code operates under the assumption that text_config should have
774
+ # `num_attention_heads` (among others). Assert here to fail early
775
+ # if transformers config doesn't align with this assumption.
776
+ assert hasattr(text_config, "num_attention_heads")
777
+
778
+ return text_config
779
780
781
def try_get_generation_config(
vllm/worker/cpu_model_runner.py
+3
-8
@@ -508,13 +508,8 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
508
logger.warning("Regarding multimodal models, vLLM currently "
509
"only supports adding LoRA to language model.")
510
511
- # It's necessary to distinguish between the max_position_embeddings
512
- # of VLMs and LLMs.
513
- if hasattr(self.model.config, "max_position_embeddings"):
514
- max_pos_embeddings = self.model.config.max_position_embeddings
515
- else:
516
- max_pos_embeddings = (
517
- self.model.config.text_config.max_position_embeddings)
511
+ # Use get_text_config() in case of multimodal models
512
+ text_config = self.model_config.hf_config.get_text_config()
513
514
self.lora_manager = LRUCacheWorkerLoRAManager(
515
self.scheduler_config.max_num_seqs,
@@ -524,7 +519,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
519
self.device,
520
self.model.embedding_modules,
521
self.model.embedding_padding_modules,
527
- max_position_embeddings=max_pos_embeddings,
522
+ max_position_embeddings=text_config.max_position_embeddings,
523
)
524
self.model = self.lora_manager.create_lora_manager(self.model)
525
vllm/worker/hpu_model_runner.py
+5
-9
@@ -724,14 +724,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
724
"Bias support in LoRA is not enabled in HPU yet."
725
assert not self.lora_config.fully_sharded_loras, \
726
"Fully sharded LoRAs is not enabled in HPU yet."
727
- # It's necessary to distinguish between the
728
- # max_position_embeddings of VLMs and LLMs.
729
- if hasattr(self.model.config, "max_position_embeddings"):
730
- max_pos_embeddings = (
731
- self.model.config.max_position_embeddings)
732
- else:
733
- max_pos_embeddings = (
734
- self.model.config.text_config.max_position_embeddings)
727
+
728
+ # Use get_text_config() in case of multimodal models
729
+ text_config = self.model_config.hf_config.get_text_config()
730
731
self.lora_manager = LRUCacheWorkerLoRAManager(
732
self.scheduler_config.max_num_seqs,
@@ -741,7 +736,8 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
736
self.device,
737
self.model.embedding_modules,
738
self.model.embedding_padding_modules,
744
- max_position_embeddings=max_pos_embeddings,
739
+ max_position_embeddings=text_config.
740
+ max_position_embeddings,
741
)
742
self.model = self.lora_manager.create_lora_manager(self.model)
743
vllm/worker/model_runner.py
+5
-9
@@ -1130,14 +1130,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1130
logger.warning(
1131
"Regarding multimodal models, vLLM currently "
1132
"only supports adding LoRA to language model.")
1133
- # It's necessary to distinguish between the
1134
- # max_position_embeddings of VLMs and LLMs.
1135
- if hasattr(self.model.config, "max_position_embeddings"):
1136
- max_pos_embeddings = (
1137
- self.model.config.max_position_embeddings)
1138
- else:
1139
- max_pos_embeddings = (
1140
- self.model.config.text_config.max_position_embeddings)
1133
+
1134
+ # Use get_text_config() in case of multimodal models
1135
+ text_config = self.model_config.hf_config.get_text_config()
1136
1137
self.lora_manager = LRUCacheWorkerLoRAManager(
1138
self.scheduler_config.max_num_seqs,
@@ -1147,7 +1142,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1142
self.device,
1143
self.model.embedding_modules,
1144
self.model.embedding_padding_modules,
1150
- max_position_embeddings=max_pos_embeddings,
1145
+ max_position_embeddings=text_config.
1146
+ max_position_embeddings,
1147
)
1148
self.model = self.lora_manager.create_lora_manager(self.model)
1149
time_after_load = time.perf_counter()