A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] Add support for embedding model GritLM (#10816)
Signed-off-by: Pooya Davoodi <pooya.davoodi@parasail.io>
Pooya Davoodi committed
Dec 11, 2024 at 22:39 UTC
1da8f0e1dddaf8625829e7ecca7fce93eb685c03
5 files changed
+458
docs/source/models/supported_models.rst
+10
@@ -203,6 +203,11 @@ Text Generation (``--task generate``)
203
- :code:`ibm-granite/granite-3.0-1b-a400m-base`, :code:`ibm-granite/granite-3.0-3b-a800m-instruct`, :code:`ibm/PowerMoE-3b`, etc.
204
- ✅︎
205
- ✅︎
206
+ * - :code:`GritLM`
207
+ - GritLM
208
+ - :code:`parasail-ai/GritLM-7B-vllm`.
209
+ - ✅︎
210
+ - ✅︎
211
* - :code:`InternLMForCausalLM`
212
- InternLM
213
- :code:`internlm/internlm-7b`, :code:`internlm/internlm-chat-7b`, etc.
@@ -395,6 +400,11 @@ The following table lists those that are tested in vLLM.
400
- :code:`BAAI/bge-multilingual-gemma2`, etc.
401
-
402
- ✅︎
403
+ * - :code:`GritLM`
404
+ - GritLM
405
+ - :code:`parasail-ai/GritLM-7B-vllm`.
406
+ - ✅︎
407
+ - ✅︎
408
* - :code:`LlamaModel`, :code:`LlamaForCausalLM`, :code:`MistralModel`, etc.
409
- Llama-based
410
- :code:`intfloat/e5-mistral-7b-instruct`, etc.
tests/models/embedding/language/test_gritlm.py
new
+200
@@ -0,0 +1,200 @@
1
+import importlib.util
2
+import math
3
+from array import array
4
+from typing import List
5
+
6
+import openai
7
+import pytest
8
+import pytest_asyncio
9
+from scipy.spatial.distance import cosine
10
+
11
+import vllm
12
+import vllm.config
13
+
14
+from ....utils import RemoteOpenAIServer
15
+
16
+# GritLM embedding implementation is only supported by XFormers backend.
17
+pytest.mark.skipif(not importlib.util.find_spec("xformers"),
18
+ reason="GritLM requires XFormers")
19
+
20
+MODEL_NAME = "parasail-ai/GritLM-7B-vllm"
21
+MAX_MODEL_LEN = 4000
22
+
23
+
24
+def _arr(arr):
25
+ """
26
+ Convert a list of integers to an array of integers.
27
+ """
28
+ return array("i", arr)
29
+
30
+
31
+def test_find_array(monkeypatch):
32
+ # GritLM embedding implementation is only supported by XFormers backend.
33
+ monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
34
+
35
+ from vllm.model_executor.models.gritlm import GritLMPooler
36
+
37
+ # Create an LLM object to get the model config.
38
+ llm = vllm.LLM(MODEL_NAME, task="embedding", max_model_len=MAX_MODEL_LEN)
39
+ pooler = GritLMPooler(model_config=llm.llm_engine.model_config)
40
+
41
+ arr = _arr([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
42
+
43
+ assert pooler._find_array(arr, _arr([3, 4, 5]), start_idx=0) == 3
44
+ assert pooler._find_array(arr, _arr([3, 4, 5]), start_idx=1) == 3
45
+ assert pooler._find_array(arr, _arr([3, 4, 5]), start_idx=5) == -1
46
+ assert pooler._find_array(arr, _arr([3, 5]), start_idx=0) == -1
47
+
48
+ with pytest.raises(ValueError):
49
+ pooler._find_array(arr, _arr([3, 4, 5]), start_idx=-1)
50
+
51
+
52
+@pytest.fixture(scope="module")
53
+def server_embedding():
54
+ # GritLM embedding implementation is only supported by XFormers backend.
55
+ with pytest.MonkeyPatch.context() as mp:
56
+ mp.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
57
+
58
+ args = ["--task", "embedding", "--max_model_len", str(MAX_MODEL_LEN)]
59
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
60
+ yield remote_server
61
+
62
+
63
+@pytest.fixture(scope="module")
64
+def server_generate():
65
+ args = ["--task", "generate", "--max_model_len", str(MAX_MODEL_LEN)]
66
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
67
+ yield remote_server
68
+
69
+
70
+@pytest_asyncio.fixture
71
+async def client_embedding(server_embedding: RemoteOpenAIServer):
72
+ async with server_embedding.get_async_client() as async_client:
73
+ yield async_client
74
+
75
+
76
+@pytest_asyncio.fixture
77
+async def client_generate(server_generate: RemoteOpenAIServer):
78
+ async with server_generate.get_async_client() as async_client:
79
+ yield async_client
80
+
81
+
82
+def run_llm_encode(llm: vllm.LLM, queries: List[str],
83
+ instruction: str) -> List[float]:
84
+ outputs = llm.encode([instruction + q for q in queries], )
85
+ return [output.outputs.embedding for output in outputs]
86
+
87
+
88
+async def run_client_embeddings(client: vllm.LLM, queries: List[str],
89
+ instruction: str) -> List[float]:
90
+ outputs = await client.embeddings.create(
91
+ model=MODEL_NAME,
92
+ input=[instruction + q for q in queries],
93
+ )
94
+ return [data.embedding for data in outputs.data]
95
+
96
+
97
+def gritlm_instruction(instruction):
98
+ return ("<|user|>\n" + instruction +
99
+ "\n<|embed|>\n" if instruction else "<|embed|>\n")
100
+
101
+
102
+def get_test_data():
103
+ """
104
+ Grabbed this test data and the expected values from
105
+ README.md in https://github.com/ContextualAI/gritlm
106
+ """
107
+ q_instruction = gritlm_instruction(
108
+ "Given a scientific paper title, retrieve the paper's abstract")
109
+ queries = [
110
+ "Bitcoin: A Peer-to-Peer Electronic Cash System",
111
+ "Generative Representational Instruction Tuning",
112
+ ]
113
+
114
+ d_instruction = gritlm_instruction("")
115
+ documents = [
116
+ # ruff: noqa: E501
117
+ "A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution. Digital signatures provide part of the solution, but the main benefits are lost if a trusted third party is still required to prevent double-spending. We propose a solution to the double-spending problem using a peer-to-peer network. The network timestamps transactions by hashing them into an ongoing chain of hash-based proof-of-work, forming a record that cannot be changed without redoing the proof-of-work. The longest chain not only serves as proof of the sequence of events witnessed, but proof that it came from the largest pool of CPU power. As long as a majority of CPU power is controlled by nodes that are not cooperating to attack the network, they'll generate the longest chain and outpace attackers. The network itself requires minimal structure. Messages are broadcast on a best effort basis, and nodes can leave and rejoin the network at will, accepting the longest proof-of-work chain as proof of what happened while they were gone.",
118
+ "All text-based language problems can be reduced to either generation or embedding. Current models only perform well at one or the other. We introduce generative representational instruction tuning (GRIT) whereby a large language model is trained to handle both generative and embedding tasks by distinguishing between them through instructions. Compared to other open models, our resulting GritLM 7B sets a new state of the art on the Massive Text Embedding Benchmark (MTEB) and outperforms all models up to its size on a range of generative tasks. By scaling up further, GritLM 8X7B outperforms all open generative language models that we tried while still being among the best embedding models. Notably, we find that GRIT matches training on only generative or embedding data, thus we can unify both at no performance loss. Among other benefits, the unification via GRIT speeds up Retrieval-Augmented Generation (RAG) by > 60% for long documents, by no longer requiring separate retrieval and generation models. Models, code, etc. are freely available at https://github.com/ContextualAI/gritlm.",
119
+ ]
120
+
121
+ return queries, q_instruction, documents, d_instruction
122
+
123
+
124
+def validate_embed_output(q_rep: List[float], d_rep: List[float]):
125
+ cosine_sim_q0_d0 = 1 - cosine(q_rep[0], d_rep[0])
126
+ assert math.isclose(cosine_sim_q0_d0, 0.609, abs_tol=0.001)
127
+
128
+ cosine_sim_q0_d1 = 1 - cosine(q_rep[0], d_rep[1])
129
+ assert math.isclose(cosine_sim_q0_d1, 0.101, abs_tol=0.001)
130
+
131
+ cosine_sim_q1_d0 = 1 - cosine(q_rep[1], d_rep[0])
132
+ assert math.isclose(cosine_sim_q1_d0, 0.120, abs_tol=0.001)
133
+
134
+ cosine_sim_q1_d1 = 1 - cosine(q_rep[1], d_rep[1])
135
+ assert math.isclose(cosine_sim_q1_d1, 0.532, abs_tol=0.001)
136
+
137
+
138
+def test_gritlm_offline_embedding(monkeypatch):
139
+ # GritLM embedding implementation is only supported by XFormers backend.
140
+ monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
141
+
142
+ queries, q_instruction, documents, d_instruction = get_test_data()
143
+
144
+ llm = vllm.LLM(MODEL_NAME, task="embedding", max_model_len=MAX_MODEL_LEN)
145
+
146
+ d_rep = run_llm_encode(
147
+ llm,
148
+ documents,
149
+ d_instruction,
150
+ )
151
+ q_rep = run_llm_encode(
152
+ llm,
153
+ queries,
154
+ q_instruction,
155
+ )
156
+
157
+ validate_embed_output(q_rep, d_rep)
158
+
159
+
160
+@pytest.mark.asyncio
161
+async def test_gritlm_api_server_embedding(
162
+ client_embedding: openai.AsyncOpenAI):
163
+ queries, q_instruction, documents, d_instruction = get_test_data()
164
+
165
+ d_rep = await run_client_embeddings(
166
+ client_embedding,
167
+ documents,
168
+ d_instruction,
169
+ )
170
+ q_rep = await run_client_embeddings(
171
+ client_embedding,
172
+ queries,
173
+ q_instruction,
174
+ )
175
+
176
+ validate_embed_output(q_rep, d_rep)
177
+
178
+
179
+def test_gritlm_offline_gen():
180
+ input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
181
+
182
+ llm = vllm.LLM(MODEL_NAME, max_model_len=MAX_MODEL_LEN)
183
+ sampling_params = vllm.SamplingParams(temperature=0.0, max_tokens=256)
184
+ outputs = llm.generate(input, sampling_params=sampling_params)
185
+
186
+ assert outputs[0].outputs[0].text == "The capital of France is Paris."
187
+
188
+
189
+@pytest.mark.asyncio
190
+async def test_gritlm_api_server_gen(client_generate: openai.AsyncOpenAI):
191
+ input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
192
+
193
+ outputs = await client_generate.completions.create(
194
+ model=MODEL_NAME,
195
+ prompt=input,
196
+ max_tokens=256,
197
+ temperature=0.0,
198
+ )
199
+
200
+ assert outputs.choices[0].text == "The capital of France is Paris."
tests/models/registry.py
+1
@@ -135,6 +135,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
135
# [Text-only]
136
"BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
137
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
138
+ "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
139
"LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
140
"MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
141
"Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
vllm/model_executor/models/gritlm.py
new
+245
@@ -0,0 +1,245 @@
1
+from array import array
2
+from typing import List, Optional, Union
3
+
4
+import torch
5
+from torch import nn
6
+from xformers.ops.fmha.attn_bias import BlockDiagonalMask
7
+
8
+from vllm.attention import AttentionMetadata
9
+from vllm.attention.backends.xformers import XFormersImpl
10
+from vllm.config import ModelConfig, VllmConfig
11
+from vllm.logger import init_logger
12
+from vllm.model_executor.models.llama import LlamaForCausalLM
13
+from vllm.model_executor.pooling_metadata import (PoolingMetadata,
14
+ PoolingTensors)
15
+from vllm.multimodal.utils import cached_get_tokenizer
16
+from vllm.sequence import (EmbeddingSequenceGroupOutput, IntermediateTensors,
17
+ PoolerOutput)
18
+
19
+logger = init_logger(__name__)
20
+
21
+
22
+class GritLMPooler(nn.Module):
23
+
24
+ def __init__(self, model_config: ModelConfig):
25
+ super().__init__()
26
+
27
+ self.model_config = model_config
28
+
29
+ tokenizer = cached_get_tokenizer(
30
+ self.model_config.tokenizer,
31
+ tokenizer_mode=self.model_config.tokenizer_mode,
32
+ tokenizer_revision=self.model_config.tokenizer_revision,
33
+ trust_remote_code=self.model_config.trust_remote_code,
34
+ )
35
+
36
+ # Collect the tokens needed for pattern matching.
37
+ # "▁<" is different from "_<". The former uses "▁" to indicate that
38
+ # the next token is the start of a word.
39
+ # "<0x0A>" is the newline token (i.e. "\n")."
40
+ self.token_ids = {
41
+ tok: tokenizer.convert_tokens_to_ids([tok])[0]
42
+ for tok in ["<s>", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
43
+ }
44
+
45
+ def tokens_to_ids(tokens: list[str]) -> array:
46
+ return array("i", [self.token_ids[token] for token in tokens])
47
+
48
+ self.user_pattern_ids = tokens_to_ids(
49
+ ["▁<", "|", "user", "|", ">", "<0x0A>"])
50
+ self.embed_newline_pattern_ids = tokens_to_ids(
51
+ ["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"])
52
+ self.embed_pattern_ids = tokens_to_ids(
53
+ ["▁<", "|", "embed", "|", ">", "<0x0A>"])
54
+
55
+ def _find_array(self, arr: array, target: array, start_idx: int) -> int:
56
+ """
57
+ Find the first occurrence of target in arr starting from start_idx.
58
+
59
+ Args:
60
+ arr: The array to search within
61
+ target: The consecutive subsequence to find
62
+ start_idx: The starting index to search from
63
+
64
+ Returns:
65
+ int: The index of the first occurrence of target in arr.
66
+ """
67
+ if start_idx < 0:
68
+ raise ValueError("start_idx must be non-negative")
69
+ if not target or not arr:
70
+ raise ValueError("Empty arr or target not allowed")
71
+
72
+ target_len = len(target)
73
+ for i in range(start_idx, len(arr) - target_len + 1):
74
+ if arr[i:i + target_len] == target:
75
+ return i
76
+ return -1
77
+
78
+ def _get_instruction_len(self, prompt_token_ids: array) -> bool:
79
+ """
80
+ Get the length of the instruction in the prompt.
81
+
82
+ We do a pattern matching to find the instruction in the prompt,
83
+ and then return the length of the instruction.
84
+
85
+ The pattern matching is done using integers instead of strings
86
+ because the prompt is given as a list of token IDs.
87
+ """
88
+
89
+ instruction_len = 0
90
+
91
+ # Return no instruction in case of missing BOS token.
92
+ if prompt_token_ids[0] != self.token_ids["<s>"]:
93
+ logger.warning("BOS token not found in prompt,"
94
+ "thus using empty string for instruction."
95
+ "GritLM requires BOS token in prompt.")
96
+ return instruction_len
97
+
98
+ # If user pattern is found in the prompt, that means there should be
99
+ # a newline token before the embed pattern.
100
+ embed_pattern_ids = self.embed_pattern_ids
101
+ if self._find_array(prompt_token_ids,
102
+ self.user_pattern_ids,
103
+ start_idx=1) == 1:
104
+ embed_pattern_ids = self.embed_newline_pattern_ids
105
+
106
+ # Find the embed pattern in the prompt.
107
+ found_embed_pattern_idx = self._find_array(prompt_token_ids,
108
+ embed_pattern_ids,
109
+ start_idx=1)
110
+
111
+ if found_embed_pattern_idx != -1:
112
+ instruction_len = found_embed_pattern_idx + len(embed_pattern_ids)
113
+ else:
114
+ logger.warning("Query instruction not found in prompt,"
115
+ "thus using BOS token as instruction instead."
116
+ "GritLM requires query instruction in prompt.")
117
+ instruction_len = 1
118
+
119
+ return instruction_len
120
+
121
+ def forward(
122
+ self,
123
+ hidden_states: torch.Tensor,
124
+ pooling_metadata: PoolingMetadata,
125
+ ) -> PoolerOutput:
126
+ """
127
+ Pool the hidden states by summing the embeddings of
128
+ non-instruction tokens.
129
+ """
130
+ prompts_token_ids = [
131
+ token_ids.prompt_token_ids_array
132
+ for _, token_ids in pooling_metadata.seq_data.items()
133
+ ]
134
+
135
+ instruction_lens = torch.tensor(
136
+ [
137
+ self._get_instruction_len(prompt_token_ids)
138
+ for prompt_token_ids in prompts_token_ids
139
+ ],
140
+ device=hidden_states.device,
141
+ )
142
+
143
+ prompt_lens = PoolingTensors.from_pooling_metadata(
144
+ pooling_metadata, hidden_states.device).prompt_lens
145
+
146
+ mask = torch.zeros_like(hidden_states, dtype=torch.bool)
147
+
148
+ start_idx = 0
149
+ for prompt_len, instruction_len in zip(prompt_lens, instruction_lens):
150
+ end_idx = start_idx + prompt_len
151
+ mask[start_idx + instruction_len:end_idx] = True
152
+ start_idx = end_idx
153
+
154
+ masked_hidden_states = hidden_states.masked_fill(~mask, 0.0)
155
+
156
+ sum_embeddings = torch.zeros(len(prompt_lens),
157
+ hidden_states.size(1),
158
+ device=hidden_states.device)
159
+
160
+ start_idx = 0
161
+ for i, prompt_len in enumerate(prompt_lens):
162
+ end_idx = start_idx + prompt_len
163
+ sum_embeddings[i] = masked_hidden_states[start_idx:end_idx].sum(
164
+ dim=0)
165
+ start_idx = end_idx
166
+
167
+ num_non_instruction_tokens = prompt_lens - instruction_lens
168
+ mean_embeddings = sum_embeddings / num_non_instruction_tokens.unsqueeze(
169
+ 1)
170
+
171
+ pooled_data = nn.functional.normalize(mean_embeddings, p=2, dim=1)
172
+
173
+ pooled_outputs = [
174
+ EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
175
+ ]
176
+
177
+ return PoolerOutput(outputs=pooled_outputs)
178
+
179
+
180
+class GritLM(LlamaForCausalLM):
181
+ """This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
182
+
183
+ The class inherits from LlamaForCausalLM and provides a custom pooling
184
+ layer.
185
+
186
+ The main difference between the pooling layer in GritLM and the one in
187
+ LlamaForCausalLM is that GritLM ignores the query instruction in the prompt
188
+ when pooling the hidden states.
189
+
190
+ Embedding prompts should be in the following format:
191
+ - With instruction: "<|user|>\nINSTRUCTION\n<|embed|>\nPROMPT".
192
+ - Without instruction: "<|embed|>\nPROMPT".
193
+
194
+ Generation prompts should be in the following format:
195
+ - "<|user|>\nPROMPT\n<|assistant|>\n"
196
+ """
197
+
198
+ def __init__(
199
+ self,
200
+ vllm_config: VllmConfig,
201
+ prefix: str = "",
202
+ **kwargs,
203
+ ) -> None:
204
+ super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
205
+
206
+ self.task = vllm_config.model_config.task
207
+
208
+ self._pooler = GritLMPooler(vllm_config.model_config)
209
+
210
+ for layer in self.model.layers:
211
+ if self.task == "embedding" and hasattr(layer, "self_attn"):
212
+ assert isinstance(layer.self_attn.attn.impl, XFormersImpl), (
213
+ "GritLM embedding is only supported by XFormers backend, "
214
+ "which can be forced by VLLM_ATTENTION_BACKEND=XFORMERS")
215
+
216
+ def forward(
217
+ self,
218
+ input_ids: torch.Tensor,
219
+ positions: torch.Tensor,
220
+ kv_caches: List[torch.Tensor],
221
+ attn_metadata: AttentionMetadata,
222
+ **kwargs,
223
+ ) -> Union[torch.Tensor, IntermediateTensors]:
224
+
225
+ # Change attention to non-causal for embedding task.
226
+ if self.task == "embedding":
227
+ assert attn_metadata.prefill_metadata.attn_bias is None
228
+ attn_metadata.prefill_metadata.attn_bias = [
229
+ BlockDiagonalMask.from_seqlens(attn_metadata.seq_lens)
230
+ ]
231
+
232
+ return super().forward(
233
+ input_ids=input_ids,
234
+ positions=positions,
235
+ kv_caches=kv_caches,
236
+ attn_metadata=attn_metadata,
237
+ **kwargs,
238
+ )
239
+
240
+ def pooler(
241
+ self,
242
+ hidden_states: torch.Tensor,
243
+ pooling_metadata: PoolingMetadata,
244
+ ) -> Optional[PoolerOutput]:
245
+ return self._pooler(hidden_states, pooling_metadata)
vllm/model_executor/models/registry.py
+2
@@ -56,6 +56,7 @@ _TEXT_GENERATION_MODELS = {
56
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
57
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
58
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
59
+ "GritLM": ("gritlm", "GritLM"),
60
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
61
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
62
"InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
@@ -110,6 +111,7 @@ _EMBEDDING_MODELS = {
111
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
112
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
113
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
114
+ "GritLM": ("gritlm", "GritLM"),
115
"LlamaModel": ("llama", "LlamaForCausalLM"),
116
**{
117
# Multiple models share the same architecture, so we include them all