@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd89ce16d2

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 22:45 UTC

[Model] Introduce verify_and_update_model_config for VerifyAndUpdateConfig. (#31131)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com>

wang.yuqi committed Dec 24, 2025 at 17:54 UTC bd89ce16d216f33d93cb72d0a88b2a98d726784a

3 files changed +47 -30

vllm/config/model.py

+18 -1

@@ -595,7 +595,7 @@ class ModelConfig:

595

596 # Avoid running try_verify_and_update_config multiple times

597 self.config_updated = False

598 -

598 + self._try_verify_and_update_model_config()

599 self._verify_quantization()

600 self._verify_cuda_graph()

601 self._verify_bnb_config()

@@ -1008,6 +1008,23 @@ class ModelConfig:

1008 "when expert parallelism is enabled."

1009 )

1010

1011 + def _try_verify_and_update_model_config(self):

1012 + # Avoid running try_verify_and_update_config multiple times

1013 + if getattr(self, "config_updated", False):

1014 + return

1015 +

1016 + architecture = self.architecture

1017 + if architecture is None:

1018 + return

1019 +

1020 + from vllm.model_executor.models.config import (

1021 + MODELS_CONFIG_MAP,

1022 + )

1023 +

1024 + cls = MODELS_CONFIG_MAP.get(architecture, None)

1025 + if cls is not None:

1026 + cls.verify_and_update_model_config(self)

1027 +

1028 def verify_dual_chunk_attention_config(

1029 self,

1030 load_config: LoadConfig,

vllm/model_executor/models/config.py

+29 -26

@@ -13,7 +13,7 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE

13 from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec, MLAAttentionSpec

15 if TYPE_CHECKING:

16 - from vllm.config import VllmConfig

16 + from vllm.config import ModelConfig, VllmConfig

18 logger = init_logger(__name__)

@@ -21,20 +21,24 @@ logger = init_logger(__name__)

21 class VerifyAndUpdateConfig:

22 @staticmethod

23 def verify_and_update_config(vllm_config: "VllmConfig") -> None:

24 - raise NotImplementedError

24 + return

25 +

26 + @staticmethod

27 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

28 + return

27 -class Gemma3TextModelConfig:

31 +class Gemma3TextModelConfig(VerifyAndUpdateConfig):

32 @staticmethod

29 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

30 - hf_config = vllm_config.model_config.hf_config

33 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

34 + hf_config = model_config.hf_config

35 hf_config.is_causal = not hf_config.use_bidirectional_attention

38 class GteNewModelConfig(VerifyAndUpdateConfig):

39 @staticmethod

36 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

37 - config = vllm_config.model_config.hf_config

40 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

41 + config = model_config.hf_config

43 assert config.__class__.__name__ == "NewConfig"

44 assert config.hidden_act == "gelu"

@@ -53,16 +57,15 @@ class GteNewModelConfig(VerifyAndUpdateConfig):

58 class JambaForSequenceClassificationConfig(VerifyAndUpdateConfig):

59 @staticmethod

56 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

57 - pooler_config = vllm_config.model_config.pooler_config

60 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

61 + pooler_config = model_config.pooler_config

62 if pooler_config.use_activation is None:

63 pooler_config.use_activation = False

66 class JinaRobertaModelConfig(VerifyAndUpdateConfig):

67 @staticmethod

64 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

65 - model_config = vllm_config.model_config

68 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

69 config = model_config.hf_config

71 if config.position_embedding_type == "rotary":

@@ -90,10 +93,10 @@ class JinaRobertaModelConfig(VerifyAndUpdateConfig):

94 class LlamaBidirectionalConfig(VerifyAndUpdateConfig):

95 @staticmethod

93 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

96 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

97 from vllm.config.pooler import PoolingTypeStr

96 - hf_config = vllm_config.model_config.hf_config

99 + hf_config = model_config.hf_config

100 hf_config.is_causal = False

101

102 pooling_type_map: dict[str, PoolingTypeStr] = {

@@ -105,7 +108,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):

108 pooling_type = pooling_type_map.get(hf_config.pooling, None)

109 if pooling_type is None:

110 raise ValueError(f"pool_type {hf_config.pooling} not supported")

108 - vllm_config.model_config.pooler_config.pooling_type = pooling_type

111 + model_config.pooler_config.pooling_type = pooling_type

112

113

114 class NomicBertModelConfig(VerifyAndUpdateConfig):

@@ -204,8 +207,8 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):

207

208 class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig):

209 @staticmethod

207 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

208 - pooler_config = vllm_config.model_config.pooler_config

210 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

211 + pooler_config = model_config.pooler_config

212

213 if pooler_config.step_tag_id is None:

214 pooler_config.step_tag_id = 151651

@@ -213,8 +216,8 @@ class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig):

216

217 class Qwen2ForRewardModelConfig(VerifyAndUpdateConfig):

218 @staticmethod

216 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

217 - pooler_config = vllm_config.model_config.pooler_config

219 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

220 + pooler_config = model_config.pooler_config

221

222 if pooler_config.softmax is None:

223 pooler_config.softmax = False

@@ -222,8 +225,8 @@ class Qwen2ForRewardModelConfig(VerifyAndUpdateConfig):

225

226 class Qwen3ForSequenceClassificationConfig(VerifyAndUpdateConfig):

227 @staticmethod

225 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

226 - config = vllm_config.model_config.hf_config

228 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

229 + config = model_config.hf_config

230

231 is_original_qwen3_reranker = getattr(

232 config, "is_original_qwen3_reranker", False

@@ -237,23 +240,23 @@ class Qwen3ForSequenceClassificationConfig(VerifyAndUpdateConfig):

240 "Try loading the original Qwen3 Reranker?, see: "

241 "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/offline_reranker.py"

242 )

240 - vllm_config.model_config.hf_config.method = "from_2_way_softmax"

243 + model_config.hf_config.method = "from_2_way_softmax"

244

245

246 class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig):

247 @staticmethod

245 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

246 - config = vllm_config.model_config.hf_config

248 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

249 + config = model_config.hf_config

250 config.num_labels = 1

248 - pooler_config = vllm_config.model_config.pooler_config

251 + pooler_config = model_config.pooler_config

252 if pooler_config.logit_bias is None:

253 pooler_config.logit_bias = 2.65

254

255

256 class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):

257 @staticmethod

255 - def verify_and_update_config(vllm_config: "VllmConfig") -> None:

256 - config = vllm_config.model_config.hf_config

258 + def verify_and_update_model_config(model_config: "ModelConfig") -> None:

259 + config = model_config.hf_config

260

261 assert config.__class__.__name__ == "GteConfig"

262 assert config.hidden_act == "gelu"

vllm/model_executor/models/llama.py

-3

@@ -64,7 +64,6 @@ from .interfaces import (

64 SupportsLoRA,

65 SupportsPP,

66 )

67 -from .interfaces_base import attn_type

67 from .utils import (

68 AutoWeightsLoader,

69 PPMissingLayer,

@@ -707,14 +706,12 @@ class LlamaForCausalLM(

706 return name, loaded_weight

707

708

710 -@attn_type("encoder_only")

709 class LlamaBidirectionalForSequenceClassification(as_seq_cls_model(LlamaForCausalLM)):

710 # This class sets the correct attention type and pooling type

711 # through LlamaBidirectionalConfig.

712 pass

713

714

717 -@attn_type("encoder_only")

715 class LlamaBidirectionalModel(as_embedding_model(LlamaForCausalLM)):

716 # This class sets the correct attention type and pooling type

717 # through LlamaBidirectionalConfig.