A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix hidden_size for multimodal classification model (#24501)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Sep 10, 2025 at 01:37 UTC
9ad0688e436a41e386fa49e81ee344cb59f7d23c
2 files changed
+10
-2
vllm/model_executor/models/adapters.py
+3
-2
@@ -255,7 +255,7 @@ def as_seq_cls_model(cls: _T) -> _T:
255
from vllm.model_executor.models.interfaces import SupportsCrossEncoding
256
from vllm.sequence import IntermediateTensors
257
258
- from .utils import maybe_prefix
258
+ from .utils import get_model_hidden_size, maybe_prefix
259
260
class ModelForSequenceClassification(_create_pooling_model_cls(cls),
261
SupportsCrossEncoding):
@@ -263,9 +263,10 @@ def as_seq_cls_model(cls: _T) -> _T:
263
def _init_pooler(self, vllm_config: "VllmConfig", prefix: str = ""):
264
config = vllm_config.model_config.hf_config
265
quant_config = vllm_config.quant_config
266
+ hidden_size = get_model_hidden_size(config)
267
268
self.score = ReplicatedLinear(
268
- config.hidden_size,
269
+ hidden_size,
270
config.num_labels,
271
bias=False,
272
params_dtype=torch.float32,
vllm/model_executor/models/utils.py
+7
@@ -761,3 +761,10 @@ def fast_topk(values: torch.Tensor, topk: int,
761
else:
762
# Use topk for efficiency with larger k values
763
return torch.topk(values, topk, dim=dim)
764
+
765
+
766
+def get_model_hidden_size(hf_config: PretrainedConfig) -> int:
767
+ if hasattr(hf_config, "hidden_size"):
768
+ return hf_config.hidden_size
769
+ text_config = hf_config.get_text_config()
770
+ return text_config.hidden_size