@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f465ab533
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[FEATURE] Enables offline /score for embedding models (#12021)

Signed-off-by: Gabriel Marinho <gmarinho@ibm.com>

Gabriel Marinho committed Jan 28, 2025 at 00:30 UTC 0f465ab53303fbd3c8ad32163db161cdb0cf8dad
2 files changed +216 -44
tests/models/embedding/language/test_scoring.py
+100
@@ -5,12 +5,18 @@ Run `pytest tests/models/embedding/language/test_scoring.py`.
5 import math
6
7 import pytest
8 +import torch
9 +import torch.nn.functional as F
10
11 MODELS = [
12 "cross-encoder/ms-marco-MiniLM-L-6-v2", # Bert
13 "BAAI/bge-reranker-v2-m3", # Roberta
14 ]
15
16 +EMBEDDING_MODELS = [
17 + "sentence-transformers/all-MiniLM-L12-v2",
18 +]
19 +
20 TEXTS_1 = [
21 "What is the capital of France?",
22 "What is the capital of Germany?",
@@ -87,3 +93,97 @@ def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
93
94 assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
95 assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
96 +
97 +
98 +@pytest.fixture(scope="module", params=EMBEDDING_MODELS)
99 +def emb_model_name(request):
100 + yield request.param
101 +
102 +
103 +@pytest.mark.parametrize("dtype", ["half"])
104 +def test_llm_1_to_1_embedding(vllm_runner, hf_runner, emb_model_name,
105 + dtype: str):
106 +
107 + text_pair = [TEXTS_1[0], TEXTS_2[0]]
108 +
109 + with hf_runner(emb_model_name, dtype=dtype,
110 + is_sentence_transformer=True) as hf_model:
111 + hf_embeddings = hf_model.encode(text_pair)
112 + hf_outputs = [
113 + F.cosine_similarity(*map(torch.tensor, hf_embeddings), dim=0)
114 + ]
115 +
116 + with vllm_runner(emb_model_name,
117 + task="embed",
118 + dtype=dtype,
119 + max_model_len=None) as vllm_model:
120 + vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
121 +
122 + assert len(vllm_outputs) == 1
123 + assert len(hf_outputs) == 1
124 +
125 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
126 +
127 +
128 +@pytest.mark.parametrize("dtype", ["half"])
129 +def test_llm_1_to_N_embedding(vllm_runner, hf_runner, emb_model_name,
130 + dtype: str):
131 +
132 + text_pairs = [
133 + [TEXTS_1[0], TEXTS_2[0]],
134 + [TEXTS_1[0], TEXTS_2[1]],
135 + ]
136 +
137 + with hf_runner(emb_model_name, dtype=dtype,
138 + is_sentence_transformer=True) as hf_model:
139 + hf_embeddings = [
140 + hf_model.encode(text_pair) for text_pair in text_pairs
141 + ]
142 + hf_outputs = [
143 + F.cosine_similarity(*map(torch.tensor, pair), dim=0)
144 + for pair in hf_embeddings
145 + ]
146 +
147 + with vllm_runner(emb_model_name,
148 + task="embed",
149 + dtype=dtype,
150 + max_model_len=None) as vllm_model:
151 + vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
152 +
153 + assert len(vllm_outputs) == 2
154 + assert len(hf_outputs) == 2
155 +
156 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
157 + assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
158 +
159 +
160 +@pytest.mark.parametrize("dtype", ["half"])
161 +def test_llm_N_to_N_embedding(vllm_runner, hf_runner, emb_model_name,
162 + dtype: str):
163 +
164 + text_pairs = [
165 + [TEXTS_1[0], TEXTS_2[0]],
166 + [TEXTS_1[1], TEXTS_2[1]],
167 + ]
168 +
169 + with hf_runner(emb_model_name, dtype=dtype,
170 + is_sentence_transformer=True) as hf_model:
171 + hf_embeddings = [
172 + hf_model.encode(text_pair) for text_pair in text_pairs
173 + ]
174 + hf_outputs = [
175 + F.cosine_similarity(*map(torch.tensor, pair), dim=0)
176 + for pair in hf_embeddings
177 + ]
178 +
179 + with vllm_runner(emb_model_name,
180 + task="embed",
181 + dtype=dtype,
182 + max_model_len=None) as vllm_model:
183 + vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
184 +
185 + assert len(vllm_outputs) == 2
186 + assert len(hf_outputs) == 2
187 +
188 + assert math.isclose(hf_outputs[0], vllm_outputs[0], rel_tol=0.01)
189 + assert math.isclose(hf_outputs[1], vllm_outputs[1], rel_tol=0.01)
vllm/entrypoints/llm.py
+116 -44
@@ -5,6 +5,7 @@ from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5 Tuple, Type, Union, cast, overload)
6
7 import cloudpickle
8 +import torch
9 import torch.nn as nn
10 from tqdm import tqdm
11 from typing_extensions import TypeVar, deprecated
@@ -996,6 +997,107 @@ class LLM:
997
998 return [ClassificationRequestOutput.from_base(item) for item in items]
999
1000 + def _embedding_score(
1001 + self,
1002 + tokenizer: AnyTokenizer,
1003 + text_1: List[Union[str, TextPrompt, TokensPrompt]],
1004 + text_2: List[Union[str, TextPrompt, TokensPrompt]],
1005 + truncate_prompt_tokens: Optional[int] = None,
1006 + use_tqdm: bool = True,
1007 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
1008 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1009 + ) -> List[ScoringRequestOutput]:
1010 +
1011 + encoded_output = self.encode(
1012 + text_1 + text_2,
1013 + use_tqdm=use_tqdm,
1014 + lora_request=lora_request,
1015 + prompt_adapter_request=prompt_adapter_request)
1016 + encoded_output_1 = encoded_output[0:len(text_1)]
1017 + encoded_output_2 = encoded_output[len(text_1):]
1018 +
1019 + if len(encoded_output_1) == 1:
1020 + encoded_output_1 = encoded_output_1 * len(encoded_output_2)
1021 +
1022 + output_pairs = [(t1, t2)
1023 + for t1, t2 in zip(encoded_output_1, encoded_output_2)]
1024 +
1025 + scores = []
1026 + scorer = torch.nn.CosineSimilarity(0)
1027 +
1028 + for embed_1, embed_2 in output_pairs:
1029 + pair_score = scorer(embed_1.outputs.data, embed_2.outputs.data)
1030 +
1031 + if (pad_token_id := getattr(tokenizer, "pad_token_id",
1032 + None)) is not None:
1033 + tokens = embed_1.prompt_token_ids + [
1034 + pad_token_id
1035 + ] + embed_2.prompt_token_ids
1036 + else:
1037 + tokens = embed_1.prompt_token_ids + embed_2.prompt_token_ids
1038 +
1039 + scores.append(
1040 + PoolingRequestOutput(
1041 + request_id=f"{embed_1.request_id}_{embed_2.request_id}",
1042 + outputs=pair_score,
1043 + prompt_token_ids=tokens,
1044 + finished=True))
1045 +
1046 + items = self.engine_class.validate_outputs(scores,
1047 + PoolingRequestOutput)
1048 + return [ScoringRequestOutput.from_base(item) for item in items]
1049 +
1050 + def _cross_encoding_score(
1051 + self,
1052 + tokenizer: Union[AnyTokenizer],
1053 + text_1: List[Union[str, TextPrompt, TokensPrompt]],
1054 + text_2: List[Union[str, TextPrompt, TokensPrompt]],
1055 + truncate_prompt_tokens: Optional[int] = None,
1056 + use_tqdm: bool = True,
1057 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
1058 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1059 + ) -> List[ScoringRequestOutput]:
1060 +
1061 + if isinstance(tokenizer, MistralTokenizer):
1062 + raise ValueError(
1063 + "Score API is only enabled for `--task embed or score`")
1064 +
1065 + if len(text_1) == 1:
1066 + text_1 = text_1 * len(text_2)
1067 +
1068 + input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
1069 +
1070 + pooling_params = PoolingParams()
1071 +
1072 + tokenization_kwargs: Dict[str, Any] = {}
1073 + if truncate_prompt_tokens is not None:
1074 + tokenization_kwargs["truncation"] = True
1075 + tokenization_kwargs["max_length"] = truncate_prompt_tokens
1076 +
1077 + parsed_prompts = []
1078 +
1079 + for q, t in input_pairs:
1080 + prompt_inputs = tokenizer(text=q,
1081 + text_pair=t,
1082 + **tokenization_kwargs)
1083 + engine_prompt = TokensPrompt(
1084 + prompt_token_ids=prompt_inputs["input_ids"],
1085 + token_type_ids=prompt_inputs.get("token_type_ids"))
1086 + parsed_prompts.append(engine_prompt)
1087 +
1088 + self._validate_and_add_requests(
1089 + prompts=parsed_prompts,
1090 + params=pooling_params,
1091 + lora_request=lora_request,
1092 + prompt_adapter_request=prompt_adapter_request,
1093 + )
1094 +
1095 + outputs = self._run_engine(use_tqdm=use_tqdm)
1096 + items = self.engine_class.validate_outputs(outputs,
1097 + PoolingRequestOutput)
1098 +
1099 + return [ScoringRequestOutput.from_base(item) for item in items]
1100 +
1101 def score(
1102 self,
1103 text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
@@ -1047,25 +1149,20 @@ class LLM:
1149
1150 raise ValueError(" ".join(messages))
1151
1050 - if not self.llm_engine.model_config.is_cross_encoder:
1051 - raise ValueError("Your model does not support cross encoding")
1052 - if self.llm_engine.model_config.task != "score":
1053 - raise ValueError("Score API is only enabled for `--task score`")
1054 -
1055 - tokenizer = self.llm_engine.get_tokenizer()
1056 -
1057 - if isinstance(tokenizer, MistralTokenizer):
1152 + if self.llm_engine.model_config.task not in ("embed", "score"):
1153 raise ValueError(
1059 - "MistralTokenizer not supported for cross-encoding")
1154 + "Score API is only enabled for `--task embed or --task score`")
1155
1156 # the tokenizer for models such as
1157 # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
1158 # lists of tokens to the `text` and `text_pair` kwargs
1159 + tokenizer = self.llm_engine.get_tokenizer()
1160 +
1161 def ensure_str(prompt: SingletonPrompt):
1162 if isinstance(prompt, dict):
1163 if "multi_modal_data" in prompt:
1164 raise ValueError("Multi-modal prompt is not "
1068 - "supported for cross encoding")
1165 + "supported for scoring")
1166 elif "prompt_token_ids" in prompt:
1167 prompt = tokenizer.decode(
1168 cast(TokensPrompt, prompt)["prompt_token_ids"])
@@ -1091,40 +1188,15 @@ class LLM:
1188 if len(text_2) == 0:
1189 raise ValueError("At least one text_pair element must be given")
1190
1094 - if len(text_1) == 1:
1095 - text_1 = text_1 * len(text_2)
1096 -
1097 - input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
1098 - pooling_params = PoolingParams()
1099 -
1100 - tokenization_kwargs: Dict[str, Any] = {}
1101 - if truncate_prompt_tokens is not None:
1102 - tokenization_kwargs["truncation"] = True
1103 - tokenization_kwargs["max_length"] = truncate_prompt_tokens
1104 -
1105 - parsed_prompts = []
1106 -
1107 - for q, t in input_pairs:
1108 - prompt_inputs = tokenizer(text=q,
1109 - text_pair=t,
1110 - **tokenization_kwargs)
1111 - engine_prompt = TokensPrompt(
1112 - prompt_token_ids=prompt_inputs["input_ids"],
1113 - token_type_ids=prompt_inputs.get("token_type_ids"))
1114 - parsed_prompts.append(engine_prompt)
1115 -
1116 - self._validate_and_add_requests(
1117 - prompts=parsed_prompts,
1118 - params=pooling_params,
1119 - lora_request=lora_request,
1120 - prompt_adapter_request=prompt_adapter_request,
1121 - )
1122 -
1123 - outputs = self._run_engine(use_tqdm=use_tqdm)
1124 - items = self.engine_class.validate_outputs(outputs,
1125 - PoolingRequestOutput)
1126 -
1127 - return [ScoringRequestOutput.from_base(item) for item in items]
1191 + if self.llm_engine.model_config.is_cross_encoder:
1192 + return self._cross_encoding_score(tokenizer, text_1, text_2,
1193 + truncate_prompt_tokens, use_tqdm,
1194 + lora_request,
1195 + prompt_adapter_request)
1196 + else:
1197 + return self._embedding_score(tokenizer, text_1, text_2,
1198 + truncate_prompt_tokens, use_tqdm,
1199 + lora_request, prompt_adapter_request)
1200
1201 def start_profile(self) -> None:
1202 self.llm_engine.start_profile()