A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Model][1/N] Automatic conversion of CrossEncoding model (#20012)
Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Jun 27, 2025 at 12:10 UTC
cd4cfee68902dcad9498b3d9d4530b817499d592
5 files changed
+239
-167
tests/models/language/pooling/mteb_utils.py
+7
-4
@@ -43,7 +43,7 @@ class VllmMtebEncoder(mteb.Encoder):
43
# issues by randomizing the order.
44
r = self.rng.permutation(len(sentences))
45
sentences = [sentences[i] for i in r]
46
- outputs = self.model.encode(sentences, use_tqdm=False)
46
+ outputs = self.model.embed(sentences, use_tqdm=False)
47
embeds = np.array(outputs)
48
embeds = embeds[np.argsort(r)]
49
return embeds
@@ -250,16 +250,19 @@ def mteb_test_rerank_models(hf_runner,
250
with vllm_runner(model_info.name,
251
task="score",
252
max_model_len=None,
253
+ max_num_seqs=8,
254
**vllm_extra_kwargs) as vllm_model:
255
256
+ model_config = vllm_model.model.llm_engine.model_config
257
+
258
if model_info.architecture:
256
- assert (model_info.architecture
257
- in vllm_model.model.llm_engine.model_config.architectures)
259
+ assert (model_info.architecture in model_config.architectures)
260
+ assert model_config.hf_config.num_labels == 1
261
262
vllm_main_score = run_mteb_rerank(VllmMtebEncoder(vllm_model),
263
tasks=MTEB_RERANK_TASKS,
264
languages=MTEB_RERANK_LANGS)
262
- vllm_dtype = vllm_model.model.llm_engine.model_config.dtype
265
+ vllm_dtype = model_config.dtype
266
267
with hf_runner(model_info.name, is_cross_encoder=True,
268
dtype="float32") as hf_model:
vllm/config.py
+28
-1
@@ -569,6 +569,10 @@ class ModelConfig:
569
else:
570
self.truncation_side = "right"
571
572
+ model_info, arch = self.registry.inspect_model_cls(self.architectures)
573
+ self._model_info = model_info
574
+ self._architecture = arch
575
+
576
self.pooler_config = self._init_pooler_config()
577
578
self.dtype = _get_and_verify_dtype(
@@ -660,8 +664,18 @@ class ModelConfig:
664
665
@property
666
def architectures(self) -> list[str]:
667
+ # architectures in the model config.
668
return getattr(self.hf_config, "architectures", [])
669
670
+ @property
671
+ def architecture(self) -> str:
672
+ # The architecture vllm actually used.
673
+ return self._architecture
674
+
675
+ @property
676
+ def model_info(self) -> dict[str, Any]:
677
+ return self._model_info
678
+
679
def maybe_pull_model_tokenizer_for_s3(self, model: str,
680
tokenizer: str) -> None:
681
"""Pull model/tokenizer from S3 to temporary directory when needed.
@@ -4450,6 +4464,9 @@ class VllmConfig:
4464
def __post_init__(self):
4465
"""Verify configs are valid & consistent with each other.
4466
"""
4467
+
4468
+ self.try_verify_and_update_config()
4469
+
4470
if self.model_config is not None:
4471
self.model_config.verify_async_output_proc(self.parallel_config,
4472
self.speculative_config,
@@ -4694,11 +4711,21 @@ class VllmConfig:
4711
batch_size_capture_list)
4712
4713
def recalculate_max_model_len(self, max_model_len: int):
4714
+ # Can only be called in try_verify_and_update_config
4715
model_config = self.model_config
4716
max_model_len = model_config.get_and_verify_max_len(max_model_len)
4717
self.model_config.max_model_len = max_model_len
4718
self.scheduler_config.max_model_len = max_model_len
4701
- self.compute_hash()
4719
+
4720
+ def try_verify_and_update_config(self):
4721
+ architecture = getattr(self.model_config, "architecture", None)
4722
+ if architecture is None:
4723
+ return
4724
+
4725
+ from vllm.model_executor.models.config import MODELS_CONFIG_MAP
4726
+ cls = MODELS_CONFIG_MAP.get(architecture, None)
4727
+ if cls is not None:
4728
+ cls.verify_and_update_config(self)
4729
4730
def __str__(self):
4731
return (
vllm/model_executor/models/bert_with_rope.py
+1
-148
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Iterable
4
-from copy import deepcopy
4
from typing import Optional
5
6
import torch
@@ -12,7 +11,6 @@ from vllm.attention import Attention, AttentionType
11
from vllm.compilation.decorators import support_torch_compile
12
from vllm.config import CacheConfig, VllmConfig
13
from vllm.distributed import get_tensor_model_parallel_world_size
15
-from vllm.logger import init_logger
14
from vllm.model_executor.layers.activation import (get_act_and_mul_fn,
15
get_act_fn)
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -30,8 +28,6 @@ from vllm.model_executor.models.interfaces import SupportsQuant
28
from vllm.model_executor.models.utils import WeightsMapper
29
from vllm.sequence import IntermediateTensors
30
33
-logger = init_logger(__name__)
34
-
31
32
class BertWithRopeEmbedding(nn.Module):
33
@@ -408,7 +404,7 @@ class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):
404
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
405
super().__init__()
406
self.vllm_config = vllm_config
411
- self.config = self.config_verify(vllm_config)
407
+ self.config = vllm_config.model_config.hf_config
408
self.embeddings = BertWithRopeEmbedding(self.config)
409
self.encoder = BertWithRopeEncoder(
410
vllm_config=vllm_config,
@@ -416,9 +412,6 @@ class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):
412
rotary_kwargs=self.config.rotary_kwargs,
413
prefix=f"{prefix}.encoder")
414
419
- def config_verify(self, vllm_config):
420
- raise NotImplementedError
421
-
415
def forward(
416
self,
417
input_ids: Optional[torch.Tensor],
@@ -490,95 +483,6 @@ class NomicBertModel(BertWithRope):
483
"norm2": "mlp_ln",
484
})
485
493
- def config_verify(self, vllm_config):
494
- config = vllm_config.model_config.hf_config
495
-
496
- assert config.__class__.__name__ == "NomicBertConfig"
497
- assert config.activation_function in ["swiglu", "gelu"]
498
- config.position_embedding_type = getattr(config,
499
- "position_embedding_type",
500
- "rope")
501
-
502
- if config.activation_function == "swiglu":
503
- config.hidden_act = "silu"
504
- else:
505
- config.hidden_act = config.activation_function
506
-
507
- assert (config.mlp_fc1_bias == config.mlp_fc2_bias ==
508
- config.qkv_proj_bias)
509
- config.bias = config.qkv_proj_bias
510
-
511
- assert config.rotary_emb_scale_base is None
512
- assert not config.rotary_emb_interleaved
513
-
514
- config.layer_norm_eps = config.layer_norm_epsilon
515
- config.intermediate_size = config.n_inner
516
- config.hidden_size = config.n_embd
517
- config.num_hidden_layers = config.n_layer
518
-
519
- head_dim = config.hidden_size // config.num_attention_heads
520
- rotary_emb_dim = head_dim * config.rotary_emb_fraction
521
- max_trained_positions = getattr(config, "max_trained_positions", 2048)
522
- config.rotary_kwargs = {
523
- "head_size": head_dim,
524
- "rotary_dim": rotary_emb_dim,
525
- "max_position": max_trained_positions,
526
- "base": getattr(config, "rope_theta", config.rotary_emb_base),
527
- "rope_scaling": getattr(config, "rope_scaling", None)
528
- }
529
-
530
- # we ignore config.rotary_scaling_factor so that for datasets shorter
531
- # than max_trained_positions 2048, the results are consistent
532
- # with SentenceTransformer.
533
- # The context extension uses vllm style rope_theta and rope_scaling.
534
- # See #17785 #18755
535
- if (not vllm_config.model_config.hf_overrides
536
- and vllm_config.model_config.original_max_model_len is None):
537
- # Default
538
- # Reset max_model_len to max_trained_positions.
539
- # nomic-embed-text-v2-moe the length is set to 512
540
- # by sentence_bert_config.json.
541
- max_model_len_before = vllm_config.model_config.max_model_len
542
- max_model_len = min(vllm_config.model_config.max_model_len,
543
- max_trained_positions)
544
-
545
- vllm_config.recalculate_max_model_len(max_model_len)
546
- logger.warning(
547
- "Nomic context extension is disabled. "
548
- "Changing max_model_len from %s to %s. "
549
- "To enable context extension, see: "
550
- "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/context_extension.html",
551
- max_model_len_before, vllm_config.model_config.max_model_len)
552
- else:
553
- # We need to re-verify max_model_len to avoid lengths
554
- # greater than position_embedding.
555
- model_config = vllm_config.model_config
556
- hf_text_config = model_config.hf_text_config
557
-
558
- if isinstance(model_config.hf_overrides, dict):
559
- # hf_overrides_kw
560
- max_model_len = model_config.hf_overrides.get(
561
- "max_model_len", vllm_config.model_config.max_model_len)
562
- else:
563
- # hf_overrides_fn
564
- # This might be overridden by sentence_bert_config.json.
565
- max_model_len = vllm_config.model_config.max_model_len
566
-
567
- # reset hf_text_config for recalculate_max_model_len.
568
- if hasattr(hf_text_config, "max_model_len"):
569
- delattr(hf_text_config, "max_model_len")
570
- hf_text_config.max_position_embeddings = max_trained_positions
571
- hf_text_config.rope_scaling = config.rotary_kwargs["rope_scaling"]
572
-
573
- # The priority of sentence_bert_config.json is higher
574
- # than max_position_embeddings
575
- encoder_config = deepcopy(model_config.encoder_config)
576
- encoder_config.pop("max_seq_length", None)
577
- model_config.encoder_config = encoder_config
578
-
579
- vllm_config.recalculate_max_model_len(max_model_len)
580
- return config
581
-
486
487
class GteNewModel(BertWithRope):
488
# for https://huggingface.co/Alibaba-NLP/new-impl
@@ -600,24 +504,6 @@ class GteNewModel(BertWithRope):
504
layer.mlp.gate_up_proj.bias = None
505
layer.mlp.gate_up_proj.skip_bias_add = True
506
603
- def config_verify(self, vllm_config):
604
- config = vllm_config.model_config.hf_config
605
-
606
- assert config.__class__.__name__ == "NewConfig"
607
- assert config.hidden_act == "gelu"
608
-
609
- config.hidden_act = "geglu"
610
-
611
- head_dim = config.hidden_size // config.num_attention_heads
612
- config.rotary_kwargs = {
613
- "head_size": head_dim,
614
- "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
615
- "max_position": config.max_position_embeddings,
616
- "base": config.rope_theta,
617
- "rope_scaling": getattr(config, "rope_scaling", None)
618
- }
619
- return config
620
-
507
def split_up_gate_proj(self, weights: Iterable[tuple[str, torch.Tensor]]):
508
n = "mlp.up_gate_proj"
509
for name, weight in weights:
@@ -652,24 +538,6 @@ class SnowflakeGteNewModel(GteNewModel):
538
"attention.o_proj": "attn.out_proj",
539
})
540
655
- def config_verify(self, vllm_config):
656
- config = vllm_config.model_config.hf_config
657
-
658
- assert config.__class__.__name__ == "GteConfig"
659
- assert config.hidden_act == "gelu"
660
-
661
- config.hidden_act = "geglu"
662
-
663
- head_dim = config.hidden_size // config.num_attention_heads
664
- config.rotary_kwargs = {
665
- "head_size": head_dim,
666
- "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
667
- "max_position": config.max_position_embeddings,
668
- "base": config.rope_theta,
669
- "rope_scaling": getattr(config, "rope_scaling", None)
670
- }
671
- return config
672
-
541
542
class JinaRobertaModel(BertWithRope):
543
# for https://huggingface.co/jinaai/jina-embeddings-v3
@@ -685,21 +553,6 @@ class JinaRobertaModel(BertWithRope):
553
"norm2": "mlp_ln",
554
})
555
688
- def config_verify(self, vllm_config):
689
- config = vllm_config.model_config.hf_config
690
-
691
- assert config.__class__.__name__ == "XLMRobertaFlashConfig"
692
-
693
- head_dim = config.hidden_size // config.num_attention_heads
694
- config.rotary_kwargs = {
695
- "head_size": head_dim,
696
- "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
697
- "max_position": config.max_position_embeddings,
698
- "base": getattr(config, "rope_theta", config.rotary_emb_base),
699
- "rope_scaling": getattr(config, "rope_scaling", None)
700
- }
701
- return config
702
-
556
def forward(
557
self,
558
input_ids: torch.Tensor,
vllm/model_executor/models/config.py
new
+200
@@ -0,0 +1,200 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from copy import deepcopy
4
+from typing import TYPE_CHECKING
5
+
6
+from vllm.logger import init_logger
7
+
8
+if TYPE_CHECKING:
9
+ from vllm.config import VllmConfig
10
+
11
+logger = init_logger(__name__)
12
+
13
+
14
+class VerifyAndUpdateConfig:
15
+
16
+ @staticmethod
17
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
18
+ raise NotImplementedError
19
+
20
+
21
+class GteNewModelConfig(VerifyAndUpdateConfig):
22
+
23
+ @staticmethod
24
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
25
+ config = vllm_config.model_config.hf_config
26
+
27
+ assert config.__class__.__name__ == "NewConfig"
28
+ assert config.hidden_act == "gelu"
29
+
30
+ config.hidden_act = "geglu"
31
+
32
+ head_dim = config.hidden_size // config.num_attention_heads
33
+ config.rotary_kwargs = {
34
+ "head_size": head_dim,
35
+ "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
36
+ "max_position": config.max_position_embeddings,
37
+ "base": config.rope_theta,
38
+ "rope_scaling": getattr(config, "rope_scaling", None)
39
+ }
40
+
41
+
42
+class JinaRobertaModelConfig(VerifyAndUpdateConfig):
43
+
44
+ @staticmethod
45
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
46
+ config = vllm_config.model_config.hf_config
47
+
48
+ if config.position_embedding_type == "rotary":
49
+ assert config.__class__.__name__ == "XLMRobertaFlashConfig"
50
+
51
+ head_dim = config.hidden_size // config.num_attention_heads
52
+ config.rotary_kwargs = {
53
+ "head_size": head_dim,
54
+ "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
55
+ "max_position": config.max_position_embeddings,
56
+ "base": getattr(config, "rope_theta", config.rotary_emb_base),
57
+ "rope_scaling": getattr(config, "rope_scaling", None)
58
+ }
59
+
60
+
61
+class NomicBertModelConfig(VerifyAndUpdateConfig):
62
+
63
+ @staticmethod
64
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
65
+ config = vllm_config.model_config.hf_config
66
+
67
+ assert config.__class__.__name__ == "NomicBertConfig"
68
+ assert config.activation_function in ["swiglu", "gelu"]
69
+ config.position_embedding_type = getattr(config,
70
+ "position_embedding_type",
71
+ "rope")
72
+
73
+ if config.activation_function == "swiglu":
74
+ config.hidden_act = "silu"
75
+ else:
76
+ config.hidden_act = config.activation_function
77
+
78
+ assert (config.mlp_fc1_bias == config.mlp_fc2_bias ==
79
+ config.qkv_proj_bias)
80
+ config.bias = config.qkv_proj_bias
81
+
82
+ assert config.rotary_emb_scale_base is None
83
+ assert not config.rotary_emb_interleaved
84
+
85
+ config.layer_norm_eps = config.layer_norm_epsilon
86
+ config.intermediate_size = config.n_inner
87
+ config.hidden_size = config.n_embd
88
+ config.num_hidden_layers = config.n_layer
89
+
90
+ head_dim = config.hidden_size // config.num_attention_heads
91
+ rotary_emb_dim = head_dim * config.rotary_emb_fraction
92
+ max_trained_positions = getattr(config, "max_trained_positions", 2048)
93
+ config.rotary_kwargs = {
94
+ "head_size": head_dim,
95
+ "rotary_dim": rotary_emb_dim,
96
+ "max_position": max_trained_positions,
97
+ "base": getattr(config, "rope_theta", config.rotary_emb_base),
98
+ "rope_scaling": getattr(config, "rope_scaling", None)
99
+ }
100
+
101
+ # we ignore config.rotary_scaling_factor so that for datasets shorter
102
+ # than max_trained_positions 2048, the results are consistent
103
+ # with SentenceTransformer.
104
+ # The context extension uses vllm style rope_theta and rope_scaling.
105
+ # See #17785 #18755
106
+ if (not vllm_config.model_config.hf_overrides
107
+ and vllm_config.model_config.original_max_model_len is None):
108
+ # Default
109
+ # Reset max_model_len to max_trained_positions.
110
+ # nomic-embed-text-v2-moe the length is set to 512
111
+ # by sentence_bert_config.json.
112
+ max_model_len_before = vllm_config.model_config.max_model_len
113
+ max_model_len = min(vllm_config.model_config.max_model_len,
114
+ max_trained_positions)
115
+
116
+ vllm_config.recalculate_max_model_len(max_model_len)
117
+ logger.warning(
118
+ "Nomic context extension is disabled. "
119
+ "Changing max_model_len from %s to %s. "
120
+ "To enable context extension, see: "
121
+ "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/context_extension.html",
122
+ max_model_len_before, vllm_config.model_config.max_model_len)
123
+ else:
124
+ # We need to re-verify max_model_len to avoid lengths
125
+ # greater than position_embedding.
126
+ model_config = vllm_config.model_config
127
+ hf_text_config = model_config.hf_text_config
128
+
129
+ if isinstance(model_config.hf_overrides, dict):
130
+ # hf_overrides_kw
131
+ max_model_len = model_config.hf_overrides.get(
132
+ "max_model_len", vllm_config.model_config.max_model_len)
133
+ else:
134
+ # hf_overrides_fn
135
+ # This might be overridden by sentence_bert_config.json.
136
+ max_model_len = vllm_config.model_config.max_model_len
137
+
138
+ # reset hf_text_config for recalculate_max_model_len.
139
+ if hasattr(hf_text_config, "max_model_len"):
140
+ delattr(hf_text_config, "max_model_len")
141
+ hf_text_config.max_position_embeddings = max_trained_positions
142
+ hf_text_config.rope_scaling = config.rotary_kwargs["rope_scaling"]
143
+
144
+ # The priority of sentence_bert_config.json is higher
145
+ # than max_position_embeddings
146
+ encoder_config = deepcopy(model_config.encoder_config)
147
+ encoder_config.pop("max_seq_length", None)
148
+ model_config.encoder_config = encoder_config
149
+
150
+ vllm_config.recalculate_max_model_len(max_model_len)
151
+
152
+
153
+class Qwen3ForSequenceClassificationConfig(VerifyAndUpdateConfig):
154
+
155
+ @staticmethod
156
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
157
+ config = vllm_config.model_config.hf_config
158
+
159
+ is_original_qwen3_reranker = getattr(config,
160
+ "is_original_qwen3_reranker",
161
+ False)
162
+
163
+ if not is_original_qwen3_reranker:
164
+ return
165
+
166
+ tokens = getattr(config, "classifier_from_token", None)
167
+ assert tokens is not None and len(tokens) == 2, \
168
+ ("Try loading the original Qwen3 Reranker?, see: "
169
+ "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen3_reranker.py")
170
+ config.num_labels = 1
171
+
172
+
173
+class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
174
+
175
+ @staticmethod
176
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
177
+ config = vllm_config.model_config.hf_config
178
+
179
+ assert config.__class__.__name__ == "GteConfig"
180
+ assert config.hidden_act == "gelu"
181
+
182
+ config.hidden_act = "geglu"
183
+
184
+ head_dim = config.hidden_size // config.num_attention_heads
185
+ config.rotary_kwargs = {
186
+ "head_size": head_dim,
187
+ "rotary_dim": getattr(config, "rotary_emb_dim", head_dim),
188
+ "max_position": config.max_position_embeddings,
189
+ "base": config.rope_theta,
190
+ "rope_scaling": getattr(config, "rope_scaling", None)
191
+ }
192
+
193
+
194
+MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
195
+ "GteModel": SnowflakeGteNewModelConfig,
196
+ "GteNewModel": GteNewModelConfig,
197
+ "NomicBertModel": NomicBertModelConfig,
198
+ "Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig,
199
+ "XLMRobertaModel": JinaRobertaModelConfig,
200
+}
vllm/model_executor/models/qwen3.py
+3
-14
@@ -400,22 +400,10 @@ class Qwen3ForSequenceClassification(nn.Module, SupportsLoRA,
400
401
def load_weights_from_original_qwen3_reranker(
402
self, weights: Iterable[tuple[str, torch.Tensor]]):
403
- tokens = getattr(self.config, "classifier_from_token", None)
404
- assert tokens is not None and len(tokens) == 2, \
405
- ("Try loading the original Qwen3 Reranker?, see: "
406
- "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen3_reranker.py")
403
408
- self.config.num_labels = 1
404
model_config = self.vllm_config.model_config
410
-
405
+ tokens = getattr(self.config, "classifier_from_token", None)
406
device = self.score.weight.device
412
- self.score = RowParallelLinear(self.config.hidden_size,
413
- self.config.num_labels,
414
- quant_config=self.quant_config,
415
- input_is_parallel=False,
416
- bias=False,
417
- prefix=maybe_prefix(
418
- self.prefix, "score")).to(device)
407
408
if self.config.tie_word_embeddings:
409
self.lm_head = self.model.embed_tokens
@@ -443,5 +431,6 @@ class Qwen3ForSequenceClassification(nn.Module, SupportsLoRA,
431
self.score.weight.data.copy_(weight)
432
433
del self.lm_head
446
- loaded_weights.add("classifier.weight")
434
+ loaded_weights.add("score.weight")
435
loaded_weights.discard("lm_head.weight")
436
+ return loaded_weights