@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf73f0c95e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Enable optional prefix when loading embedding models (#10639)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 26, 2024 at 02:14 UTC cf73f0c95e09836efff876d5bfd9b9c6cc1ba06e
5 files changed +20 -13
vllm/model_executor/models/bert.py
+5 -4
@@ -14,18 +14,17 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.pooler import (CrossEncodingPooler, Pooler,
16 PoolingType)
17 -from vllm.model_executor.layers.quantization.base_config import (
18 - QuantizationConfig)
17 +from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 VocabParallelEmbedding)
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 -from vllm.model_executor.models.interfaces import SupportsCrossEncoding
21 from vllm.model_executor.pooling_metadata import PoolingMetadata
22 from vllm.sequence import IntermediateTensors, PoolerOutput
23 from vllm.transformers_utils.config import (
24 get_cross_encoder_activation_function)
25
28 -from .utils import maybe_prefix
26 +from .interfaces import SupportsCrossEncoding
27 +from .utils import WeightsMapper, maybe_prefix
28
29
30 class BertEmbedding(nn.Module):
@@ -442,6 +441,8 @@ class BertEmbeddingModel(nn.Module):
441 return self._pooler(hidden_states, pooling_metadata)
442
443 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
444 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
445 + weights = hf_to_vllm_mapper.apply(weights)
446 self.model.load_weights(weights)
447
448 def _build_model(self,
vllm/model_executor/models/gemma2.py
+3 -1
@@ -42,7 +42,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, PoolerOutput
43
44 from .interfaces import SupportsLoRA, SupportsPP
45 -from .utils import (AutoWeightsLoader, extract_layer_index,
45 +from .utils import (AutoWeightsLoader, WeightsMapper, extract_layer_index,
46 is_pp_missing_parameter,
47 make_empty_intermediate_tensors_factory, make_layers,
48 maybe_prefix)
@@ -511,4 +511,6 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
511 return self._pooler(hidden_states, pooling_metadata)
512
513 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
514 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
515 + weights = hf_to_vllm_mapper.apply(weights)
516 self.model.load_weights(weights)
vllm/model_executor/models/llama.py
+4 -1
@@ -53,7 +53,8 @@ from vllm.platforms import current_platform
53 from vllm.sequence import IntermediateTensors, PoolerOutput
54
55 from .interfaces import SupportsLoRA, SupportsPP
56 -from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
56 +from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
57 + is_pp_missing_parameter,
58 make_empty_intermediate_tensors_factory, make_layers,
59 maybe_prefix)
60
@@ -689,6 +690,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
690 return self._pooler(hidden_states, pooling_metadata)
691
692 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
693 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
694 + weights = hf_to_vllm_mapper.apply(weights)
695 self.model.load_weights(weights)
696
697 def load_kv_cache_scales(self, quantization_param_path: str) -> None:
vllm/model_executor/models/qwen2.py
+6 -6
@@ -50,7 +50,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.sequence import IntermediateTensors, PoolerOutput
51
52 from .interfaces import SupportsLoRA, SupportsPP
53 -from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
53 +from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
54 + is_pp_missing_parameter,
55 make_empty_intermediate_tensors_factory, make_layers,
56 maybe_prefix)
57
@@ -585,8 +586,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
586 ) -> Optional[PoolerOutput]:
587 return self._pooler(hidden_states, pooling_metadata)
588
588 - def load_weights(self, weights: Iterable[Tuple[str,
589 - torch.Tensor]]) -> Set[str]:
590 - loader = AutoWeightsLoader(self,
591 - ignore_unexpected_prefixes=["lm_head."])
592 - return loader.load_weights(weights)
589 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
590 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
591 + weights = hf_to_vllm_mapper.apply(weights)
592 + self.model.load_weights(weights)
vllm/model_executor/models/roberta.py
+2 -1
@@ -11,13 +11,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
11 VocabParallelEmbedding)
12 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 from vllm.model_executor.models.bert import BertEmbeddingModel, BertModel
14 -from vllm.model_executor.models.interfaces import SupportsCrossEncoding
14 from vllm.model_executor.models.utils import maybe_prefix
15 from vllm.model_executor.pooling_metadata import PoolingMetadata
16 from vllm.sequence import IntermediateTensors, PoolerOutput
17 from vllm.transformers_utils.config import (
18 get_cross_encoder_activation_function)
19
20 +from .interfaces import SupportsCrossEncoding
21 +
22
23 class RobertaEmbedding(nn.Module):
24