A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][Model] Fix inference for Hunyuan dense models (#25354)
Signed-off-by: anion <1005128408@qq.com> Signed-off-by: Anion <123177548+Anionex@users.noreply.github.com>
Anion committed
Sep 30, 2025 at 22:38 UTC
f4db5e6de19b21891724d443bbc3cefb4d2e1005
1 file changed
+59
-47
vllm/model_executor/models/hunyuan_v1.py
+59
-47
@@ -888,7 +888,7 @@ class HunYuanModel(nn.Module):
888
return loaded_params
889
890
891
-class HunYuanV1Base(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
891
+class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP):
892
packed_modules_mapping = {
893
"qkv_proj": [
894
"q_proj",
@@ -930,6 +930,56 @@ class HunYuanV1Base(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
930
else:
931
self.lm_head = PPMissingLayer()
932
933
+ def forward(
934
+ self,
935
+ input_ids: torch.Tensor,
936
+ positions: torch.Tensor,
937
+ intermediate_tensors: Optional[IntermediateTensors] = None,
938
+ inputs_embeds: Optional[torch.Tensor] = None,
939
+ ) -> Union[torch.Tensor, IntermediateTensors]:
940
+ model_output = self.model(input_ids, positions, intermediate_tensors,
941
+ inputs_embeds)
942
+ return model_output
943
+
944
+ def compute_logits(
945
+ self,
946
+ hidden_states: torch.Tensor,
947
+ ) -> Optional[torch.Tensor]:
948
+ logits = self.logits_processor(self.lm_head, hidden_states)
949
+ return logits
950
+
951
+ def make_empty_intermediate_tensors(
952
+ self, batch_size: int, dtype: torch.dtype,
953
+ device: torch.device) -> IntermediateTensors:
954
+ return IntermediateTensors({
955
+ "hidden_states":
956
+ torch.zeros((batch_size, self.config.hidden_size),
957
+ dtype=dtype,
958
+ device=device),
959
+ "residual":
960
+ torch.zeros((batch_size, self.config.hidden_size),
961
+ dtype=dtype,
962
+ device=device),
963
+ })
964
+
965
+ def load_weights(self, weights: Iterable[tuple[str,
966
+ torch.Tensor]]) -> set[str]:
967
+ loader = AutoWeightsLoader(
968
+ self,
969
+ skip_prefixes=(["lm_head."]
970
+ if self.config.tie_word_embeddings else None),
971
+ )
972
+ return loader.load_weights(weights)
973
+
974
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
975
+ return self.model.get_input_embeddings(input_ids)
976
+
977
+
978
+class HunYuanMoEV1Base(HunyuanV1ModelBase, MixtureOfExperts):
979
+
980
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
981
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
982
+
983
# Set MoE hyperparameters
984
self.expert_weights = []
985
self.num_expert_groups = 1
@@ -988,57 +1038,19 @@ class HunYuanV1Base(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
1038
moe.n_redundant_experts = self.num_redundant_experts
1039
moe.experts.update_expert_map()
1040
991
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
992
- return self.model.get_input_embeddings(input_ids)
1041
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
1042
+ return self.model.get_expert_mapping()
1043
994
- def forward(
995
- self,
996
- input_ids: torch.Tensor,
997
- positions: torch.Tensor,
998
- intermediate_tensors: Optional[IntermediateTensors] = None,
999
- inputs_embeds: Optional[torch.Tensor] = None,
1000
- ) -> Union[torch.Tensor, IntermediateTensors]:
1001
- model_output = self.model(input_ids, positions, intermediate_tensors,
1002
- inputs_embeds)
1003
- return model_output
1044
1005
- def compute_logits(
1006
- self,
1007
- hidden_states: torch.Tensor,
1008
- ) -> Optional[torch.Tensor]:
1009
- logits = self.logits_processor(self.lm_head, hidden_states)
1010
- return logits
1045
+class HunYuanDenseV1Base(HunyuanV1ModelBase):
1046
1012
- def make_empty_intermediate_tensors(
1013
- self, batch_size: int, dtype: torch.dtype,
1014
- device: torch.device) -> IntermediateTensors:
1015
- return IntermediateTensors({
1016
- "hidden_states":
1017
- torch.zeros((batch_size, self.config.hidden_size),
1018
- dtype=dtype,
1019
- device=device),
1020
- "residual":
1021
- torch.zeros((batch_size, self.config.hidden_size),
1022
- dtype=dtype,
1023
- device=device),
1024
- })
1025
-
1026
- def load_weights(self, weights: Iterable[tuple[str,
1027
- torch.Tensor]]) -> set[str]:
1028
- loader = AutoWeightsLoader(
1029
- self,
1030
- skip_prefixes=(["lm_head."]
1031
- if self.config.tie_word_embeddings else None),
1032
- )
1033
- return loader.load_weights(weights)
1034
-
1035
- def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
1036
- return self.model.get_expert_mapping()
1047
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1048
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
1049
1050
1039
-class HunYuanDenseV1ForCausalLM(HunYuanV1Base):
1051
+class HunYuanDenseV1ForCausalLM(HunYuanDenseV1Base):
1052
pass
1053
1054
1043
-class HunYuanMoEV1ForCausalLM(HunYuanV1Base):
1044
- pass
1055
+class HunYuanMoEV1ForCausalLM(HunYuanMoEV1Base):
1056
+ pass
\ No newline at end of file