A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[FEATURE] Enables offline /score for embedding models (#12021)
Signed-off-by: Gabriel Marinho <gmarinho@ibm.com>
Gabriel Marinho committed
Jan 28, 2025 at 00:30 UTC
0f465ab53303fbd3c8ad32163db161cdb0cf8dad
2 files changed
+216
-44
tests/models/embedding/language/test_scoring.py
+100
@@ -5,12 +5,18 @@ Run `pytest tests/models/embedding/language/test_scoring.py`.
5
import math
6
7
import pytest
8
+import torch
9
+import torch.nn.functional as F
10
11
MODELS = [
12
"cross-encoder/ms-marco-MiniLM-L-6-v2", # Bert
13
"BAAI/bge-reranker-v2-m3", # Roberta
14
]
15
16
+EMBEDDING_MODELS = [
17
+ "sentence-transformers/all-MiniLM-L12-v2",
18
+]
19
+
20
TEXTS_1 = [
21
"What is the capital of France?",
22
"What is the capital of Germany?",
@@ -87,3 +93,97 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
93
94
assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
95
assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
96
+
97
+
98
+@pytest.fixture(scope="module", params=EMBEDDING_MODELS)
99
+def emb_model_name(request):
100
+ yield request.param
101
+
102
+
103
+@pytest.mark.parametrize("dtype", ["half"])
104
+def test_llm_1_to_1_embedding(vllm_runner, hf_runner, emb_model_name,
105
+ dtype: str):
106
+
107
+ text_pair = [TEXTS_1[0], TEXTS_2[0]]
108
+
109
+ with hf_runner(emb_model_name, dtype=dtype,
110
+ is_sentence_transformer=True) as hf_model:
111
+ hf_embeddings = hf_model.encode(text_pair)
112
+ hf_outputs = [
113
+ F.cosine_similarity(*map(torch.tensor, hf_embeddings), dim=0)
114
+ ]
115
+
116
+ with vllm_runner(emb_model_name,
117
+ task="embed",
118
+ dtype=dtype,
119
+ max_model_len=None) as vllm_model:
120
+ vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
121
+
122
+ assert len(vllm_outputs) == 1
123
+ assert len(hf_outputs) == 1
124
+
125
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
126
+
127
+
128
+@pytest.mark.parametrize("dtype", ["half"])
129
+def test_llm_1_to_N_embedding(vllm_runner, hf_runner, emb_model_name,
130
+ dtype: str):
131
+
132
+ text_pairs = [
133
+ [TEXTS_1[0], TEXTS_2[0]],
134
+ [TEXTS_1[0], TEXTS_2[1]],
135
+ ]
136
+
137
+ with hf_runner(emb_model_name, dtype=dtype,
138
+ is_sentence_transformer=True) as hf_model:
139
+ hf_embeddings = [
140
+ hf_model.encode(text_pair) for text_pair in text_pairs
141
+ ]
142
+ hf_outputs = [
143
+ F.cosine_similarity(*map(torch.tensor, pair), dim=0)
144
+ for pair in hf_embeddings
145
+ ]
146
+
147
+ with vllm_runner(emb_model_name,
148
+ task="embed",
149
+ dtype=dtype,
150
+ max_model_len=None) as vllm_model:
151
+ vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
152
+
153
+ assert len(vllm_outputs) == 2
154
+ assert len(hf_outputs) == 2
155
+
156
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
157
+ assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
158
+
159
+
160
+@pytest.mark.parametrize("dtype", ["half"])
161
+def test_llm_N_to_N_embedding(vllm_runner, hf_runner, emb_model_name,
162
+ dtype: str):
163
+
164
+ text_pairs = [
165
+ [TEXTS_1[0], TEXTS_2[0]],
166
+ [TEXTS_1[1], TEXTS_2[1]],
167
+ ]
168
+
169
+ with hf_runner(emb_model_name, dtype=dtype,
170
+ is_sentence_transformer=True) as hf_model:
171
+ hf_embeddings = [
172
+ hf_model.encode(text_pair) for text_pair in text_pairs
173
+ ]
174
+ hf_outputs = [
175
+ F.cosine_similarity(*map(torch.tensor, pair), dim=0)
176
+ for pair in hf_embeddings
177
+ ]
178
+
179
+ with vllm_runner(emb_model_name,
180
+ task="embed",
181
+ dtype=dtype,
182
+ max_model_len=None) as vllm_model:
183
+ vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
184
+
185
+ assert len(vllm_outputs) == 2
186
+ assert len(hf_outputs) == 2
187
+
188
+ assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
189
+ assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
vllm/entrypoints/llm.py
+116
-44
@@ -5,6 +5,7 @@ from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5
Tuple, Type, Union, cast, overload)
6
7
import cloudpickle
8
+import torch
9
import torch.nn as nn
10
from tqdm import tqdm
11
from typing_extensions import TypeVar, deprecated
@@ -996,6 +997,107 @@ class LLM:
997
998
return [ClassificationRequestOutput.from_base(item) for item in items]
999
1000
+ def _embedding_score(
1001
+ self,
1002
+ tokenizer: AnyTokenizer,
1003
+ text_1: List[Union[str, TextPrompt, TokensPrompt]],
1004
+ text_2: List[Union[str, TextPrompt, TokensPrompt]],
1005
+ truncate_prompt_tokens: Optional[int] = None,
1006
+ use_tqdm: bool = True,
1007
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
1008
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1009
+ ) -> List[ScoringRequestOutput]:
1010
+
1011
+ encoded_output = self.encode(
1012
+ text_1 + text_2,
1013
+ use_tqdm=use_tqdm,
1014
+ lora_request=lora_request,
1015
+ prompt_adapter_request=prompt_adapter_request)
1016
+ encoded_output_1 = encoded_output[0:len(text_1)]
1017
+ encoded_output_2 = encoded_output[len(text_1):]
1018
+
1019
+ if len(encoded_output_1) == 1:
1020
+ encoded_output_1 = encoded_output_1 * len(encoded_output_2)
1021
+
1022
+ output_pairs = [(t1, t2)
1023
+ for t1, t2 in zip(encoded_output_1, encoded_output_2)]
1024
+
1025
+ scores = []
1026
+ scorer = torch.nn.CosineSimilarity(0)
1027
+
1028
+ for embed_1, embed_2 in output_pairs:
1029
+ pair_score = scorer(embed_1.outputs.data, embed_2.outputs.data)
1030
+
1031
+ if (pad_token_id := getattr(tokenizer, "pad_token_id",
1032
+ None)) is not None:
1033
+ tokens = embed_1.prompt_token_ids + [
1034
+ pad_token_id
1035
+ ] + embed_2.prompt_token_ids
1036
+ else:
1037
+ tokens = embed_1.prompt_token_ids + embed_2.prompt_token_ids
1038
+
1039
+ scores.append(
1040
+ PoolingRequestOutput(
1041
+ request_id=f"{embed_1.request_id}_{embed_2.request_id}",
1042
+ outputs=pair_score,
1043
+ prompt_token_ids=tokens,
1044
+ finished=True))
1045
+
1046
+ items = self.engine_class.validate_outputs(scores,
1047
+ PoolingRequestOutput)
1048
+ return [ScoringRequestOutput.from_base(item) for item in items]
1049
+
1050
+ def _cross_encoding_score(
1051
+ self,
1052
+ tokenizer: Union[AnyTokenizer],
1053
+ text_1: List[Union[str, TextPrompt, TokensPrompt]],
1054
+ text_2: List[Union[str, TextPrompt, TokensPrompt]],
1055
+ truncate_prompt_tokens: Optional[int] = None,
1056
+ use_tqdm: bool = True,
1057
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
1058
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1059
+ ) -> List[ScoringRequestOutput]:
1060
+
1061
+ if isinstance(tokenizer, MistralTokenizer):
1062
+ raise ValueError(
1063
+ "Score API is only enabled for `--task embed or score`")
1064
+
1065
+ if len(text_1) == 1:
1066
+ text_1 = text_1 * len(text_2)
1067
+
1068
+ input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
1069
+
1070
+ pooling_params = PoolingParams()
1071
+
1072
+ tokenization_kwargs: Dict[str, Any] = {}
1073
+ if truncate_prompt_tokens is not None:
1074
+ tokenization_kwargs["truncation"] = True
1075
+ tokenization_kwargs["max_length"] = truncate_prompt_tokens
1076
+
1077
+ parsed_prompts = []
1078
+
1079
+ for q, t in input_pairs:
1080
+ prompt_inputs = tokenizer(text=q,
1081
+ text_pair=t,
1082
+ **tokenization_kwargs)
1083
+ engine_prompt = TokensPrompt(
1084
+ prompt_token_ids=prompt_inputs["input_ids"],
1085
+ token_type_ids=prompt_inputs.get("token_type_ids"))
1086
+ parsed_prompts.append(engine_prompt)
1087
+
1088
+ self._validate_and_add_requests(
1089
+ prompts=parsed_prompts,
1090
+ params=pooling_params,
1091
+ lora_request=lora_request,
1092
+ prompt_adapter_request=prompt_adapter_request,
1093
+ )
1094
+
1095
+ outputs = self._run_engine(use_tqdm=use_tqdm)
1096
+ items = self.engine_class.validate_outputs(outputs,
1097
+ PoolingRequestOutput)
1098
+
1099
+ return [ScoringRequestOutput.from_base(item) for item in items]
1100
+
1101
def score(
1102
self,
1103
text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
@@ -1047,25 +1149,20 @@ class LLM:
1149
1150
raise ValueError(" ".join(messages))
1151
1050
- if not self.llm_engine.model_config.is_cross_encoder:
1051
- raise ValueError("Your model does not support cross encoding")
1052
- if self.llm_engine.model_config.task != "score":
1053
- raise ValueError("Score API is only enabled for `--task score`")
1054
-
1055
- tokenizer = self.llm_engine.get_tokenizer()
1056
-
1057
- if isinstance(tokenizer, MistralTokenizer):
1152
+ if self.llm_engine.model_config.task not in ("embed", "score"):
1153
raise ValueError(
1059
- "MistralTokenizer not supported for cross-encoding")
1154
+ "Score API is only enabled for `--task embed or --task score`")
1155
1156
# the tokenizer for models such as
1157
# "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
1158
# lists of tokens to the `text` and `text_pair` kwargs
1159
+ tokenizer = self.llm_engine.get_tokenizer()
1160
+
1161
def ensure_str(prompt: SingletonPrompt):
1162
if isinstance(prompt, dict):
1163
if "multi_modal_data" in prompt:
1164
raise ValueError("Multi-modal prompt is not "
1068
- "supported for cross encoding")
1165
+ "supported for scoring")
1166
elif "prompt_token_ids" in prompt:
1167
prompt = tokenizer.decode(
1168
cast(TokensPrompt, prompt)["prompt_token_ids"])
@@ -1091,40 +1188,15 @@ class LLM:
1188
if len(text_2) == 0:
1189
raise ValueError("At least one text_pair element must be given")
1190
1094
- if len(text_1) == 1:
1095
- text_1 = text_1 * len(text_2)
1096
-
1097
- input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
1098
- pooling_params = PoolingParams()
1099
-
1100
- tokenization_kwargs: Dict[str, Any] = {}
1101
- if truncate_prompt_tokens is not None:
1102
- tokenization_kwargs["truncation"] = True
1103
- tokenization_kwargs["max_length"] = truncate_prompt_tokens
1104
-
1105
- parsed_prompts = []
1106
-
1107
- for q, t in input_pairs:
1108
- prompt_inputs = tokenizer(text=q,
1109
- text_pair=t,
1110
- **tokenization_kwargs)
1111
- engine_prompt = TokensPrompt(
1112
- prompt_token_ids=prompt_inputs["input_ids"],
1113
- token_type_ids=prompt_inputs.get("token_type_ids"))
1114
- parsed_prompts.append(engine_prompt)
1115
-
1116
- self._validate_and_add_requests(
1117
- prompts=parsed_prompts,
1118
- params=pooling_params,
1119
- lora_request=lora_request,
1120
- prompt_adapter_request=prompt_adapter_request,
1121
- )
1122
-
1123
- outputs = self._run_engine(use_tqdm=use_tqdm)
1124
- items = self.engine_class.validate_outputs(outputs,
1125
- PoolingRequestOutput)
1126
-
1127
- return [ScoringRequestOutput.from_base(item) for item in items]
1191
+ if self.llm_engine.model_config.is_cross_encoder:
1192
+ return self._cross_encoding_score(tokenizer, text_1, text_2,
1193
+ truncate_prompt_tokens, use_tqdm,
1194
+ lora_request,
1195
+ prompt_adapter_request)
1196
+ else:
1197
+ return self._embedding_score(tokenizer, text_1, text_2,
1198
+ truncate_prompt_tokens, use_tqdm,
1199
+ lora_request, prompt_adapter_request)
1200
1201
def start_profile(self) -> None:
1202
self.llm_engine.start_profile()