@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15986f598c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Support Gemma2 embedding model (#9004)

Xin Yang committed Oct 4, 2024 at 23:57 UTC 15986f598c7b1f2969918c92f5c4cf7e28d5c0df
5 files changed +99 -3
tests/conftest.py
+1
@@ -277,6 +277,7 @@ class HfRunner:
277 SentenceTransformer(
278 model_name,
279 device="cpu",
280 + trust_remote_code=True,
281 ).to(dtype=torch_dtype))
282 else:
283 model_kwargs = model_kwargs if model_kwargs is not None else {}
tests/models/embedding/language/test_embedding.py
+10 -1
@@ -1,6 +1,6 @@
1 """Compare the outputs of HF and vLLM for Mistral models using greedy sampling.
2
3 -Run `pytest tests/models/test_llama_embedding.py`.
3 +Run `pytest tests/models/embedding/language/test_embedding.py`.
4 """
5 import pytest
6 import torch
@@ -8,6 +8,7 @@ import torch.nn.functional as F
8
9 MODELS = [
10 "intfloat/e5-mistral-7b-instruct",
11 + "BAAI/bge-multilingual-gemma2",
12 ]
13
14
@@ -28,6 +29,14 @@ def test_models(
29 model: str,
30 dtype: str,
31 ) -> None:
32 + # The example_prompts has ending "\n", for example:
33 + # "Write a short story about a robot that dreams for the first time.\n"
34 + # sentence_transformers will strip the input texts, see:
35 + # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
36 + # This makes the input_ids different between hf_model and vllm_model.
37 + # So we need to strip the input texts to avoid test failing.
38 + example_prompts = [str(s).strip() for s in example_prompts]
39 +
40 with hf_runner(model, dtype=dtype, is_embedding_model=True) as hf_model:
41 hf_outputs = hf_model.encode(example_prompts)
42
vllm/model_executor/models/gemma2.py
+5 -2
@@ -278,11 +278,14 @@ class Gemma2Model(nn.Module):
278 kv_caches: List[torch.Tensor],
279 attn_metadata: AttentionMetadata,
280 intermediate_tensors: Optional[IntermediateTensors],
281 + inputs_embeds: Optional[torch.Tensor] = None,
282 ) -> Union[torch.Tensor, IntermediateTensors]:
283 if get_pp_group().is_first_rank:
283 - hidden_states = self.embed_tokens(input_ids)
284 + if inputs_embeds is not None:
285 + hidden_states = inputs_embeds
286 + else:
287 + hidden_states = self.embed_tokens(input_ids)
288 hidden_states *= self.normalizer
285 -
289 residual = None
290 else:
291 assert intermediate_tensors is not None
vllm/model_executor/models/gemma2_embedding.py new
+82
@@ -0,0 +1,82 @@
1 +from typing import Iterable, List, Optional, Tuple
2 +
3 +import torch
4 +from torch import nn
5 +
6 +from vllm.attention import AttentionMetadata
7 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
8 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
9 +from vllm.model_executor.models.gemma2 import Gemma2Model
10 +from vllm.model_executor.pooling_metadata import PoolingMetadata
11 +from vllm.sequence import IntermediateTensors, PoolerOutput
12 +
13 +
14 +class Gemma2EmbeddingModel(nn.Module):
15 + """A model that uses Gemma2 with additional embedding functionalities.
16 +
17 + This class encapsulates the Gemma2Model and provides an interface for
18 + embedding operations and customized pooling functions.
19 +
20 + Attributes:
21 + model: An instance of Gemma2Model used for forward operations.
22 + _pooler: An instance of Pooler used for pooling operations.
23 + """
24 +
25 + def __init__(
26 + self,
27 + **kwargs,
28 + ) -> None:
29 + super().__init__()
30 + self.model = Gemma2Model(**kwargs)
31 + self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
32 +
33 + def forward(
34 + self,
35 + input_ids: Optional[torch.Tensor],
36 + positions: torch.Tensor,
37 + kv_caches: List[torch.Tensor],
38 + attn_metadata: AttentionMetadata,
39 + intermediate_tensors: Optional[IntermediateTensors] = None,
40 + inputs_embeds: Optional[torch.Tensor] = None,
41 + ) -> torch.Tensor:
42 + return self.model.forward(input_ids, positions, kv_caches,
43 + attn_metadata, intermediate_tensors,
44 + inputs_embeds)
45 +
46 + def pooler(
47 + self,
48 + hidden_states: torch.Tensor,
49 + pooling_metadata: PoolingMetadata,
50 + ) -> Optional[PoolerOutput]:
51 + return self._pooler(hidden_states, pooling_metadata)
52 +
53 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
54 + stacked_params_mapping = [
55 + # (param_name, shard_name, shard_id)
56 + ("qkv_proj", "q_proj", "q"),
57 + ("qkv_proj", "k_proj", "k"),
58 + ("qkv_proj", "v_proj", "v"),
59 + ("gate_up_proj", "gate_proj", 0),
60 + ("gate_up_proj", "up_proj", 1),
61 + ]
62 + params_dict = dict(self.model.named_parameters())
63 + for name, loaded_weight in weights:
64 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
65 + if weight_name not in name:
66 + continue
67 + name = name.replace(weight_name, param_name)
68 + # Skip loading extra bias for GPTQ models.
69 + if name.endswith(".bias") and name not in params_dict:
70 + continue
71 + param = params_dict[name]
72 + weight_loader = param.weight_loader
73 + weight_loader(param, loaded_weight, shard_id)
74 + break
75 + else:
76 + # Skip loading extra bias for GPTQ models.
77 + if name.endswith(".bias") and name not in params_dict:
78 + continue
79 + param = params_dict[name]
80 + weight_loader = getattr(param, "weight_loader",
81 + default_weight_loader)
82 + weight_loader(param, loaded_weight)
vllm/model_executor/models/registry.py
+1
@@ -83,6 +83,7 @@ _GENERATION_MODELS = {
83 _EMBEDDING_MODELS = {
84 "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
85 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
86 + "Gemma2Model": ("gemma2_embedding", "Gemma2EmbeddingModel"),
87 }
88
89 _MULTIMODAL_MODELS = {