A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Deprecation] Remove mean pooling default for `Qwen2EmbeddingModel` (#18913)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 30, 2025 at 14:53 UTC
4f4a6b844a6347a539b9dd87c39f557d91981988
3 files changed
+5
-76
docs/models/supported_models.md
+1
-1
@@ -401,7 +401,7 @@ Specified using `--task embed`.
401
402
!!! note
403
`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
404
- You should manually set mean pooling by passing `--override-pooler-config '{"pooling_type": "MEAN"}'`.
404
+ You need to manually set mean pooling by passing `--override-pooler-config '{"pooling_type": "MEAN"}'`.
405
406
!!! note
407
For `Alibaba-NLP/gte-Qwen2-*`, you need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
vllm/model_executor/models/qwen2.py
+3
-74
@@ -34,32 +34,27 @@ from vllm.attention import Attention, AttentionType
34
from vllm.compilation.decorators import support_torch_compile
35
from vllm.config import CacheConfig, VllmConfig
36
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
37
-from vllm.logger import init_logger
37
from vllm.model_executor.layers.activation import SiluAndMul
38
from vllm.model_executor.layers.layernorm import RMSNorm
39
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40
QKVParallelLinear,
41
RowParallelLinear)
42
from vllm.model_executor.layers.logits_processor import LogitsProcessor
44
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
43
from vllm.model_executor.layers.quantization import QuantizationConfig
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
46
ParallelLMHead, VocabParallelEmbedding)
47
from vllm.model_executor.model_loader.weight_utils import (
48
default_weight_loader, maybe_remap_kv_scale_name)
51
-from vllm.model_executor.pooling_metadata import PoolingMetadata
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
-from vllm.sequence import IntermediateTensors, PoolerOutput
50
+from vllm.sequence import IntermediateTensors
51
52
from .interfaces import SupportsLoRA, SupportsPP
56
-from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
57
- extract_layer_index, is_pp_missing_parameter,
53
+from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
54
+ is_pp_missing_parameter,
55
make_empty_intermediate_tensors_factory, make_layers,
56
maybe_prefix)
57
61
-logger = init_logger(__name__)
62
-
58
59
class Qwen2MLP(nn.Module):
60
@@ -499,69 +494,3 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
494
if self.config.tie_word_embeddings else None),
495
)
496
return loader.load_weights(weights)
502
-
503
-
504
-class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
505
- packed_modules_mapping = {
506
- "qkv_proj": [
507
- "q_proj",
508
- "k_proj",
509
- "v_proj",
510
- ],
511
- "gate_up_proj": [
512
- "gate_proj",
513
- "up_proj",
514
- ],
515
- }
516
-
517
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
518
-
519
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
520
- super().__init__()
521
- config = vllm_config.model_config.hf_config
522
- quant_config = vllm_config.quant_config
523
- lora_config = vllm_config.lora_config
524
- pooler_config = vllm_config.model_config.pooler_config
525
-
526
- self.config = config
527
- self.lora_config = lora_config
528
-
529
- self.quant_config = quant_config
530
- self.model = Qwen2Model(vllm_config=vllm_config,
531
- prefix=maybe_prefix(prefix, "model"))
532
-
533
- # TODO: Replace this model class with as_embedding_model(
534
- # Qwen2ForCausalLM) after changing the default pooling method
535
- if pooler_config.pooling_type is None:
536
- logger.warning(
537
- "This embedding model will default to last-token pooling in "
538
- "an upcoming version. To avoid breaking changes, you should "
539
- "pass `--override-pooler-config '{\"pooling_type\": \"MEAN\"}'`"
540
- " explicitly.")
541
-
542
- self._pooler = Pooler.from_config_with_defaults(
543
- pooler_config,
544
- pooling_type=PoolingType.MEAN,
545
- normalize=True,
546
- softmax=False)
547
-
548
- def forward(
549
- self,
550
- input_ids: torch.Tensor,
551
- positions: torch.Tensor,
552
- intermediate_tensors: Optional[IntermediateTensors] = None,
553
- ) -> torch.Tensor:
554
- return self.model(input_ids, positions, intermediate_tensors)
555
-
556
- def pooler(
557
- self,
558
- hidden_states: torch.Tensor,
559
- pooling_metadata: PoolingMetadata,
560
- ) -> Optional[PoolerOutput]:
561
- return self._pooler(hidden_states, pooling_metadata)
562
-
563
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
564
- weights = self.hf_to_vllm_mapper.apply(weights)
565
- weights = ((name, data) for name, data in weights
566
- if not name.startswith("lm_head."))
567
- self.model.load_weights(weights)
vllm/model_executor/models/registry.py
+1
-1
@@ -142,7 +142,7 @@ _EMBEDDING_MODELS = {
142
"ModernBertModel": ("modernbert", "ModernBertModel"),
143
"NomicBertModel": ("bert_with_rope", "NomicBertModel"),
144
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
145
- "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
145
+ "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
146
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
147
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
148
"Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),