A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
Support Cross encoder models (#10400)
Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Co-authored-by: Flavia Beo <flavia.beo@ibm.com>
Maximilien de Bayser committed
Nov 24, 2024 at 23:56 UTC
214efc2c3cb568e8eb3f7d234f3bd8f5bbe24795
28 files changed
+1370
-62
docs/source/serving/openai_compatible_server.md
+142
@@ -44,6 +44,148 @@ We currently support the following OpenAI APIs:
44
- This enables multi-modal inputs to be passed to embedding models, see [Using VLMs](../models/vlm.rst).
45
- *Note: You should run `vllm serve` with `--task embedding` to ensure that the model is being run in embedding mode.*
46
47
+## Score API for Cross Encoder Models
48
+
49
+vLLM supports *cross encoders models* at the **/v1/score** endpoint, which is not an OpenAI API standard endpoint. You can find the documentation for these kind of models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
50
+
51
+A ***Cross Encoder*** takes exactly two sentences / texts as input and either predicts a score or label for this sentence pair. It can for example predict the similarity of the sentence pair on a scale of 0 … 1.
52
+
53
+### Example of usage for a pair of a string and a list of texts
54
+
55
+In this case, the model will compare the first given text to each of the texts containing the list.
56
+
57
+```bash
58
+curl -X 'POST' \
59
+ 'http://127.0.0.1:8000/v1/score' \
60
+ -H 'accept: application/json' \
61
+ -H 'Content-Type: application/json' \
62
+ -d '{
63
+ "model": "BAAI/bge-reranker-v2-m3",
64
+ "text_1": "What is the capital of France?",
65
+ "text_2": [
66
+ "The capital of Brazil is Brasilia.",
67
+ "The capital of France is Paris."
68
+ ]
69
+}'
70
+```
71
+
72
+Response:
73
+
74
+```bash
75
+{
76
+ "id": "score-request-id",
77
+ "object": "list",
78
+ "created": 693570,
79
+ "model": "BAAI/bge-reranker-v2-m3",
80
+ "data": [
81
+ {
82
+ "index": 0,
83
+ "object": "score",
84
+ "score": [
85
+ 0.001094818115234375
86
+ ]
87
+ },
88
+ {
89
+ "index": 1,
90
+ "object": "score",
91
+ "score": [
92
+ 1
93
+ ]
94
+ }
95
+ ],
96
+ "usage": {}
97
+}
98
+```
99
+
100
+### Example of usage for a pair of two lists of texts
101
+
102
+In this case, the model will compare the one by one, making pairs by same index correspondent in each list.
103
+
104
+```bash
105
+curl -X 'POST' \
106
+ 'http://127.0.0.1:8000/v1/score' \
107
+ -H 'accept: application/json' \
108
+ -H 'Content-Type: application/json' \
109
+ -d '{
110
+ "model": "BAAI/bge-reranker-v2-m3",
111
+ "encoding_format": "float",
112
+ "text_1": [
113
+ "What is the capital of Brazil?",
114
+ "What is the capital of France?"
115
+ ],
116
+ "text_2": [
117
+ "The capital of Brazil is Brasilia.",
118
+ "The capital of France is Paris."
119
+ ]
120
+}'
121
+```
122
+
123
+Response:
124
+
125
+```bash
126
+{
127
+ "id": "score-request-id",
128
+ "object": "list",
129
+ "created": 693447,
130
+ "model": "BAAI/bge-reranker-v2-m3",
131
+ "data": [
132
+ {
133
+ "index": 0,
134
+ "object": "score",
135
+ "score": [
136
+ 1
137
+ ]
138
+ },
139
+ {
140
+ "index": 1,
141
+ "object": "score",
142
+ "score": [
143
+ 1
144
+ ]
145
+ }
146
+ ],
147
+ "usage": {}
148
+}
149
+```
150
+
151
+### Example of usage for a pair of two strings
152
+
153
+In this case, the model will compare the strings of texts.
154
+
155
+```bash
156
+curl -X 'POST' \
157
+ 'http://127.0.0.1:8000/v1/score' \
158
+ -H 'accept: application/json' \
159
+ -H 'Content-Type: application/json' \
160
+ -d '{
161
+ "model": "BAAI/bge-reranker-v2-m3",
162
+ "encoding_format": "float",
163
+ "text_1": "What is the capital of France?",
164
+ "text_2": "The capital of France is Paris."
165
+}'
166
+```
167
+
168
+Response:
169
+
170
+```bash
171
+{
172
+ "id": "score-request-id",
173
+ "object": "list",
174
+ "created": 693447,
175
+ "model": "BAAI/bge-reranker-v2-m3",
176
+ "data": [
177
+ {
178
+ "index": 0,
179
+ "object": "score",
180
+ "score": [
181
+ 1
182
+ ]
183
+ }
184
+ ],
185
+ "usage": {}
186
+}
187
+```
188
+
189
## Extra Parameters
190
191
vLLM supports a set of parameters that are not part of the OpenAI API.
examples/openai_cross_encoder_score.py
new
+58
@@ -0,0 +1,58 @@
1
+"""Examples Python client Score for Cross Encoder Models
2
+"""
3
+
4
+import argparse
5
+import json
6
+import pprint
7
+
8
+import requests
9
+
10
+
11
+def post_http_request(prompt: json, api_url: str) -> requests.Response:
12
+ headers = {"User-Agent": "Test Client"}
13
+ response = requests.post(api_url, headers=headers, json=prompt)
14
+ return response
15
+
16
+
17
+if __name__ == "__main__":
18
+ parser = argparse.ArgumentParser()
19
+ parser.add_argument("--host", type=str, default="localhost")
20
+ parser.add_argument("--port", type=int, default=8000)
21
+ parser.add_argument("--model", type=str, default="BAAI/bge-reranker-v2-m3")
22
+ args = parser.parse_args()
23
+ api_url = f"http://{args.host}:{args.port}/v1/score"
24
+
25
+ model_name = args.model
26
+
27
+ text_1 = "What is the capital of France?"
28
+ text_2 = [
29
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
30
+ ]
31
+ prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
32
+ score_response = post_http_request(prompt=prompt, api_url=api_url)
33
+ print("Prompt for text_1 is string and text_2 is a list:")
34
+ pprint.pprint(prompt)
35
+ print("Score Response:")
36
+ pprint.pprint(score_response.data)
37
+
38
+ text_1 = [
39
+ "What is the capital of Brazil?", "What is the capital of France?"
40
+ ]
41
+ text_2 = [
42
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
43
+ ]
44
+ prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
45
+ score_response = post_http_request(prompt=prompt, api_url=api_url)
46
+ print("Prompt for text_1 and text_2 are lists:")
47
+ pprint.pprint(prompt)
48
+ print("Score Response:")
49
+ pprint.pprint(score_response.data)
50
+
51
+ text_1 = "What is the capital of Brazil?"
52
+ text_2 = "The capital of Brazil is Brasilia."
53
+ prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
54
+ score_response = post_http_request(prompt=prompt, api_url=api_url)
55
+ print("Prompt for text_1 and text_2 are strings:")
56
+ pprint.pprint(prompt)
57
+ print("Score Response:")
58
+ pprint.pprint(score_response.data)
\ No newline at end of file
tests/conftest.py
+20
@@ -265,6 +265,7 @@ class HfRunner:
265
model_kwargs: Optional[Dict[str, Any]] = None,
266
is_embedding_model: bool = False,
267
is_sentence_transformer: bool = False,
268
+ is_cross_encoder: bool = False,
269
skip_tokenizer_init: bool = False,
270
auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,
271
postprocess_inputs: Callable[..., BatchEncoding] = identity,
@@ -282,6 +283,14 @@ class HfRunner:
283
device="cpu",
284
trust_remote_code=True,
285
).to(dtype=torch_dtype))
286
+ elif is_cross_encoder:
287
+ # Lazy init required for AMD CI
288
+ from sentence_transformers import CrossEncoder
289
+ self.model = CrossEncoder(model_name,
290
+ device="cpu",
291
+ trust_remote_code=True)
292
+ self.model.model = self.wrap_device(self.model.model)\
293
+ .to(dtype=torch_dtype)
294
else:
295
model_kwargs = model_kwargs if model_kwargs is not None else {}
296
self.model = self.wrap_device(
@@ -625,6 +634,9 @@ class HfRunner:
634
def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:
635
return self.model.encode(prompts)
636
637
+ def predict(self, prompts: List[List[str]]) -> torch.Tensor:
638
+ return self.model.predict(prompts, convert_to_tensor=True)
639
+
640
def __enter__(self):
641
return self
642
@@ -898,6 +910,14 @@ class VllmRunner:
910
req_outputs = self.model.encode(inputs)
911
return [req_output.outputs.embedding for req_output in req_outputs]
912
913
+ def score(
914
+ self,
915
+ text_1: Union[str, List[str]],
916
+ text_2: Union[str, List[str]],
917
+ ) -> List[List[float]]:
918
+ req_outputs = self.model.score(text_1, text_2)
919
+ return [req_output.outputs.embedding for req_output in req_outputs]
920
+
921
def __enter__(self):
922
return self
923
tests/entrypoints/openai/test_score.py
new
+93
@@ -0,0 +1,93 @@
1
+import pytest
2
+import requests
3
+
4
+from vllm.entrypoints.openai.protocol import ScoreResponse
5
+
6
+from ...utils import RemoteOpenAIServer
7
+
8
+MODEL_NAME = "BAAI/bge-reranker-v2-m3"
9
+
10
+
11
+@pytest.fixture(scope="module")
12
+def server():
13
+ args = [
14
+ "--enforce-eager",
15
+ ]
16
+
17
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
18
+ yield remote_server
19
+
20
+
21
+@pytest.mark.asyncio
22
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
23
+async def test_text_1_str_text_2_list(server: RemoteOpenAIServer,
24
+ model_name: str):
25
+ text_1 = "What is the capital of France?"
26
+ text_2 = [
27
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
28
+ ]
29
+
30
+ score_response = requests.post(server.url_for("v1/score"),
31
+ json={
32
+ "model": model_name,
33
+ "text_1": text_1,
34
+ "text_2": text_2,
35
+ })
36
+ score_response.raise_for_status()
37
+ score = ScoreResponse.model_validate(score_response.json())
38
+
39
+ assert score.id is not None
40
+ assert score.data is not None
41
+ assert len(score.data) == 2
42
+ assert score.data[0].score[0] <= 0.01
43
+ assert score.data[1].score[0] >= 0.9
44
+
45
+
46
+@pytest.mark.asyncio
47
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
48
+async def test_text_1_list_text_2_list(server: RemoteOpenAIServer,
49
+ model_name: str):
50
+ text_1 = [
51
+ "What is the capital of the United States?",
52
+ "What is the capital of France?"
53
+ ]
54
+ text_2 = [
55
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
56
+ ]
57
+
58
+ score_response = requests.post(server.url_for("v1/score"),
59
+ json={
60
+ "model": model_name,
61
+ "text_1": text_1,
62
+ "text_2": text_2,
63
+ })
64
+ score_response.raise_for_status()
65
+ score = ScoreResponse.model_validate(score_response.json())
66
+
67
+ assert score.id is not None
68
+ assert score.data is not None
69
+ assert len(score.data) == 2
70
+ assert score.data[0].score[0] <= 0.01
71
+ assert score.data[1].score[0] >= 0.9
72
+
73
+
74
+@pytest.mark.asyncio
75
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
76
+async def test_text_1_str_text_2_str(server: RemoteOpenAIServer,
77
+ model_name: str):
78
+ text_1 = "What is the capital of France?"
79
+ text_2 = "The capital of France is Paris."
80
+
81
+ score_response = requests.post(server.url_for("v1/score"),
82
+ json={
83
+ "model": model_name,
84
+ "text_1": text_1,
85
+ "text_2": text_2,
86
+ })
87
+ score_response.raise_for_status()
88
+ score = ScoreResponse.model_validate(score_response.json())
89
+
90
+ assert score.id is not None
91
+ assert score.data is not None
92
+ assert len(score.data) == 1
93
+ assert score.data[0].score[0] >= 0.9
tests/models/embedding/language/test_scoring.py
new
+95
@@ -0,0 +1,95 @@
1
+"""Compare the embedding outputs of HF and vLLM models.
2
+
3
+Run `pytest tests/models/embedding/language/test_embedding.py`.
4
+"""
5
+import math
6
+
7
+import pytest
8
+
9
+MODELS = [
10
+ "cross-encoder/ms-marco-MiniLM-L-6-v2", # Bert
11
+ "BAAI/bge-reranker-v2-m3", # Roberta
12
+]
13
+
14
+TEXTS_1 = [
15
+ "What is the capital of France?",
16
+ "What is the capital of Germany?",
17
+]
18
+
19
+TEXTS_2 = [
20
+ "The capital of France is Paris.",
21
+ "The capital of Germany is Berlin.",
22
+]
23
+
24
+
25
+@pytest.fixture(scope="module", params=MODELS)
26
+def model_name(request):
27
+ yield request.param
28
+
29
+
30
+@pytest.mark.parametrize("dtype", ["half"])
31
+def test_llm_1_to_1(vllm_runner, hf_runner, model_name, dtype: str):
32
+
33
+ text_pair = [TEXTS_1[0], TEXTS_2[0]]
34
+
35
+ with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
36
+ hf_outputs = hf_model.predict([text_pair]).tolist()
37
+
38
+ with vllm_runner(model_name,
39
+ task="embedding",
40
+ dtype=dtype,
41
+ max_model_len=None) as vllm_model:
42
+ vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
43
+
44
+ assert len(vllm_outputs) == 1
45
+ assert len(hf_outputs) == 1
46
+
47
+ assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
48
+
49
+
50
+@pytest.mark.parametrize("dtype", ["half"])
51
+def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
52
+
53
+ text_pairs = [
54
+ [TEXTS_1[0], TEXTS_2[0]],
55
+ [TEXTS_1[0], TEXTS_2[1]],
56
+ ]
57
+
58
+ with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
59
+ hf_outputs = hf_model.predict(text_pairs).tolist()
60
+
61
+ with vllm_runner(model_name,
62
+ task="embedding",
63
+ dtype=dtype,
64
+ max_model_len=None) as vllm_model:
65
+ vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
66
+
67
+ assert len(vllm_outputs) == 2
68
+ assert len(hf_outputs) == 2
69
+
70
+ assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
71
+ assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
72
+
73
+
74
+@pytest.mark.parametrize("dtype", ["half"])
75
+def test_llm_N_to_N(vllm_runner, hf_runner, model_name, dtype: str):
76
+
77
+ text_pairs = [
78
+ [TEXTS_1[0], TEXTS_2[0]],
79
+ [TEXTS_1[1], TEXTS_2[1]],
80
+ ]
81
+
82
+ with hf_runner(model_name, dtype=dtype, is_cross_encoder=True) as hf_model:
83
+ hf_outputs = hf_model.predict(text_pairs).tolist()
84
+
85
+ with vllm_runner(model_name,
86
+ task="embedding",
87
+ dtype=dtype,
88
+ max_model_len=None) as vllm_model:
89
+ vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
90
+
91
+ assert len(vllm_outputs) == 2
92
+ assert len(hf_outputs) == 2
93
+
94
+ assert math.isclose(hf_outputs[0], vllm_outputs[0][0], rel_tol=0.01)
95
+ assert math.isclose(hf_outputs[1], vllm_outputs[1][0], rel_tol=0.01)
tests/models/registry.py
+9
@@ -135,6 +135,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
135
"Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
136
"Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
137
"RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), # noqa: E501
138
+ "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"), # noqa: E501
139
"XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-large"),
140
# [Multimodal]
141
"LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
@@ -143,6 +144,13 @@ _EMBEDDING_EXAMPLE_MODELS = {
144
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
145
}
146
147
+_CROSS_ENCODER_EXAMPLE_MODELS = {
148
+ # [Text-only]
149
+ "BertForSequenceClassification": _HfExamplesInfo("cross-encoder/ms-marco-MiniLM-L-6-v2"), # noqa: E501
150
+ "RobertaForSequenceClassification": _HfExamplesInfo("cross-encoder/quora-roberta-base"), # noqa: E501
151
+ "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"), # noqa: E501
152
+}
153
+
154
_MULTIMODAL_EXAMPLE_MODELS = {
155
# [Decoder-only]
156
"Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
@@ -195,6 +203,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
203
_EXAMPLE_MODELS = {
204
**_TEXT_GENERATION_EXAMPLE_MODELS,
205
**_EMBEDDING_EXAMPLE_MODELS,
206
+ **_CROSS_ENCODER_EXAMPLE_MODELS,
207
**_MULTIMODAL_EXAMPLE_MODELS,
208
**_SPECULATIVE_DECODING_EXAMPLE_MODELS,
209
}
tests/models/test_registry.py
+16
-7
@@ -6,7 +6,10 @@ import torch.cuda
6
from vllm.model_executor.models import (is_embedding_model,
7
is_text_generation_model,
8
supports_multimodal)
9
-from vllm.model_executor.models.registry import (_EMBEDDING_MODELS,
9
+# yapf conflicts with isort for this block
10
+# yapf: disable
11
+from vllm.model_executor.models.registry import (_CROSS_ENCODER_MODELS,
12
+ _EMBEDDING_MODELS,
13
_MULTIMODAL_MODELS,
14
_SPECULATIVE_DECODING_MODELS,
15
_TEXT_GENERATION_MODELS,
@@ -29,22 +32,28 @@ def test_registry_imports(model_arch):
32
model_arch in _TEXT_GENERATION_MODELS
33
or model_arch in _MULTIMODAL_MODELS)
34
35
+ embedding_models = {**_EMBEDDING_MODELS, **_CROSS_ENCODER_MODELS}
36
assert is_embedding_model(model_cls) is (model_arch
33
- in _EMBEDDING_MODELS)
37
+ in embedding_models)
38
39
assert supports_multimodal(model_cls) is (model_arch
40
in _MULTIMODAL_MODELS)
41
42
43
@fork_new_process_for_each_test
40
-@pytest.mark.parametrize("model_arch,is_mm,init_cuda", [
41
- ("LlamaForCausalLM", False, False),
42
- ("MllamaForConditionalGeneration", True, False),
43
- ("LlavaForConditionalGeneration", True, True),
44
+@pytest.mark.parametrize("model_arch,is_mm,init_cuda,is_ce", [
45
+ ("LlamaForCausalLM", False, False, False),
46
+ ("MllamaForConditionalGeneration", True, False, False),
47
+ ("LlavaForConditionalGeneration", True, True, False),
48
+ ("BertForSequenceClassification", False, False, True),
49
+ ("RobertaForSequenceClassification", False, False, True),
50
+ ("XLMRobertaForSequenceClassification", False, False, True),
51
])
45
-def test_registry_is_multimodal(model_arch, is_mm, init_cuda):
52
+def test_registry_model_property(model_arch, is_mm, init_cuda, is_ce):
53
assert ModelRegistry.is_multimodal_model(model_arch) is is_mm
54
55
+ assert ModelRegistry.is_cross_encoder_model(model_arch) is is_ce
56
+
57
if init_cuda and current_platform.is_cuda_alike():
58
assert not torch.cuda.is_initialized()
59
vllm/config.py
+5
@@ -712,6 +712,11 @@ class ModelConfig:
712
def is_multimodal_model(self) -> bool:
713
return self.multimodal_config is not None
714
715
+ @property
716
+ def is_cross_encoder(self) -> bool:
717
+ architectures = getattr(self.hf_config, "architectures", [])
718
+ return ModelRegistry.is_cross_encoder_model(architectures)
719
+
720
721
class CacheConfig:
722
"""Configuration for the KV cache.
vllm/core/scheduler.py
+1
@@ -1357,6 +1357,7 @@ class Scheduler:
1357
encoder_seq_data=encoder_seq_data,
1358
cross_block_table=cross_block_table,
1359
state=seq_group.state,
1360
+ token_type_ids=seq_group.token_type_ids,
1361
# `multi_modal_data` will only be present for the 1st comm
1362
# between engine and worker.
1363
# the subsequent comms can still use delta, but
vllm/entrypoints/llm.py
+123
-1
@@ -20,7 +20,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
20
apply_mistral_chat_template,
21
parse_chat_messages,
22
resolve_chat_template_content_format)
23
-from vllm.inputs import PromptType, TextPrompt, TokensPrompt
23
+from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
24
from vllm.inputs.parse import parse_and_batch_prompt
25
from vllm.logger import init_logger
26
from vllm.lora.request import LoRARequest
@@ -817,6 +817,128 @@ class LLM:
817
return self.engine_class.validate_outputs(outputs,
818
EmbeddingRequestOutput)
819
820
+ def score(
821
+ self,
822
+ text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
823
+ text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
824
+ /,
825
+ truncate_prompt_tokens: Optional[int] = None,
826
+ use_tqdm: bool = True,
827
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
828
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
829
+ ) -> List[EmbeddingRequestOutput]:
830
+ """Generates similarity scores for all pairs <text,text_pair>.
831
+
832
+ The inputs can be 1 -> 1, 1 -> N or N -> N. In the 1 - N case
833
+ the text_1 sentence will be replicated N times to pair with the text_2
834
+ sentences. The input pairs are used to build a list of prompts for the
835
+ cross encoder model. This class automatically batches the prompts,
836
+ considering the memory constraint. For the best performance, put all
837
+ of your texts into a single list and pass it to this method.
838
+
839
+ Args:
840
+ text_1: can be a single prompt or a list of prompts, in which
841
+ case it has to have the same length as the text_2 list
842
+ text_2: The texts to pair with the query to form the input
843
+ to the LLM. See :class:`~vllm.inputs.PromptType` for
844
+ more details about the format of each prompts.
845
+ use_tqdm: Whether to use tqdm to display the progress bar.
846
+ lora_request: LoRA request to use for generation, if any.
847
+ prompt_adapter_request: Prompt Adapter request to use for
848
+ generation, if any.
849
+
850
+ Returns:
851
+ A list of ``EmbeddingRequestOutput`` objects containing the
852
+ generated scores in the same order as the input prompts.
853
+ """
854
+ task = self.llm_engine.model_config.task
855
+ if task != "embedding":
856
+ messages = ["LLM.score() is only supported for embedding models."]
857
+
858
+ supported_tasks = self.llm_engine.model_config.supported_tasks
859
+ if "embedding" in supported_tasks:
860
+ messages.append(
861
+ "Your model supports the 'embedding' task, but is "
862
+ f"currently initialized for the '{task}' task. Please "
863
+ "initialize the model using `--task embedding`.")
864
+
865
+ raise ValueError(" ".join(messages))
866
+
867
+ if not self.llm_engine.model_config.is_cross_encoder:
868
+ raise ValueError("Your model does not support the cross encoding")
869
+
870
+ tokenizer = self.llm_engine.get_tokenizer()
871
+
872
+ if isinstance(tokenizer, MistralTokenizer):
873
+ raise ValueError(
874
+ "MistralTokenizer not supported for cross-encoding")
875
+
876
+ # the tokenizer for models such as
877
+ # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
878
+ # lists of tokens to the `text` and `text_pair` kwargs
879
+ def ensure_str(prompt: SingletonPrompt):
880
+ if isinstance(prompt, dict):
881
+ if "multi_modal_data" in prompt:
882
+ raise ValueError("Multi-modal prompt is not "
883
+ "supported for cross encoding")
884
+ elif "prompt_token_ids" in prompt:
885
+ prompt = tokenizer.decode(
886
+ cast(TokensPrompt, prompt)["prompt_token_ids"])
887
+ elif "prompt" in prompt:
888
+ prompt = cast(TextPrompt, prompt)["prompt"]
889
+ assert type(prompt) is str
890
+ return prompt
891
+
892
+ if isinstance(text_1, (str, dict)):
893
+ # Convert a single prompt to a list.
894
+ text_1 = [text_1]
895
+ text_1 = [ensure_str(t) for t in text_1]
896
+
897
+ if isinstance(text_2, (str, dict)):
898
+ # Convert a single prompt to a list.
899
+ text_2 = [text_2]
900
+ text_2 = [ensure_str(t) for t in text_2]
901
+
902
+ if len(text_1) > 1 and len(text_1) != len(text_2):
903
+ raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
904
+ if len(text_1) == 0:
905
+ raise ValueError("At least one text element must be given")
906
+ if len(text_2) == 0:
907
+ raise ValueError("At least one text_pair element must be given")
908
+
909
+ if len(text_1) == 1:
910
+ text_1 = text_1 * len(text_2)
911
+
912
+ input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
913
+ pooling_params = PoolingParams()
914
+
915
+ tokenization_kwargs: Dict[str, Any] = {}
916
+ if truncate_prompt_tokens is not None:
917
+ tokenization_kwargs["truncation"] = True
918
+ tokenization_kwargs["max_length"] = truncate_prompt_tokens
919
+
920
+ parsed_prompts = []
921
+
922
+ for q, t in input_pairs:
923
+ prompt_inputs = tokenizer(text=q,
924
+ text_pair=t,
925
+ **tokenization_kwargs)
926
+ engine_prompt = TokensPrompt(
927
+ prompt_token_ids=prompt_inputs["input_ids"],
928
+ token_type_ids=prompt_inputs.get("token_type_ids"))
929
+ parsed_prompts.append(engine_prompt)
930
+
931
+ self._validate_and_add_requests(
932
+ prompts=parsed_prompts,
933
+ params=pooling_params,
934
+ lora_request=lora_request,
935
+ prompt_adapter_request=prompt_adapter_request,
936
+ )
937
+
938
+ outputs = self._run_engine(use_tqdm=use_tqdm)
939
+ return self.engine_class.validate_outputs(outputs,
940
+ EmbeddingRequestOutput)
941
+
942
def start_profile(self) -> None:
943
self.llm_engine.start_profile()
944
vllm/entrypoints/openai/api_server.py
+33
-2
@@ -45,6 +45,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
45
EmbeddingRequest,
46
EmbeddingResponse, ErrorResponse,
47
LoadLoraAdapterRequest,
48
+ ScoreRequest, ScoreResponse,
49
TokenizeRequest,
50
TokenizeResponse,
51
UnloadLoraAdapterRequest)
@@ -53,6 +54,7 @@ from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
54
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
55
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
56
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
57
+from vllm.entrypoints.openai.serving_score import OpenAIServingScores
58
from vllm.entrypoints.openai.serving_tokenization import (
59
OpenAIServingTokenization)
60
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -280,6 +282,10 @@ def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
282
return request.app.state.openai_serving_embedding
283
284
285
+def score(request: Request) -> Optional[OpenAIServingScores]:
286
+ return request.app.state.openai_serving_scores
287
+
288
+
289
def tokenization(request: Request) -> OpenAIServingTokenization:
290
return request.app.state.openai_serving_tokenization
291
@@ -391,6 +397,23 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
397
assert_never(generator)
398
399
400
+@router.post("/v1/score")
401
+async def create_score(request: ScoreRequest, raw_request: Request):
402
+ handler = score(raw_request)
403
+ if handler is None:
404
+ return base(raw_request).create_error_response(
405
+ message="The model does not support Score API")
406
+
407
+ generator = await handler.create_score(request, raw_request)
408
+ if isinstance(generator, ErrorResponse):
409
+ return JSONResponse(content=generator.model_dump(),
410
+ status_code=generator.code)
411
+ elif isinstance(generator, ScoreResponse):
412
+ return JSONResponse(content=generator.model_dump())
413
+
414
+ assert_never(generator)
415
+
416
+
417
if envs.VLLM_TORCH_PROFILER_DIR:
418
logger.warning(
419
"Torch Profiler is enabled in the API server. This should ONLY be "
@@ -466,8 +489,9 @@ def build_app(args: Namespace) -> FastAPI:
489
490
@app.exception_handler(RequestValidationError)
491
async def validation_exception_handler(_, exc):
469
- chat = app.state.openai_serving_chat
470
- err = chat.create_error_response(message=str(exc))
492
+ err = ErrorResponse(message=str(exc),
493
+ type="BadRequestError",
494
+ code=HTTPStatus.BAD_REQUEST)
495
return JSONResponse(err.model_dump(),
496
status_code=HTTPStatus.BAD_REQUEST)
497
@@ -565,6 +589,13 @@ def init_app_state(
589
chat_template=resolved_chat_template,
590
chat_template_content_format=args.chat_template_content_format,
591
) if model_config.task == "embedding" else None
592
+ state.openai_serving_scores = OpenAIServingScores(
593
+ engine_client,
594
+ model_config,
595
+ base_model_paths,
596
+ request_logger=request_logger
597
+ ) if (model_config.task == "embedding" \
598
+ and model_config.is_cross_encoder) else None
599
state.openai_serving_tokenization = OpenAIServingTokenization(
600
engine_client,
601
model_config,
vllm/entrypoints/openai/protocol.py
+36
@@ -806,6 +806,27 @@ class EmbeddingChatRequest(OpenAIBaseModel):
806
EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]
807
808
809
+class ScoreRequest(OpenAIBaseModel):
810
+ model: str
811
+ text_1: Union[List[str], str]
812
+ text_2: Union[List[str], str]
813
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
814
+
815
+ # doc: begin-chat-embedding-pooling-params
816
+ additional_data: Optional[Any] = None
817
+ # doc: end-chat-embedding-pooling-params
818
+
819
+ priority: int = Field(
820
+ default=0,
821
+ description=(
822
+ "The priority of the request (lower means earlier handling; "
823
+ "default: 0). Any priority other than 0 will raise an error "
824
+ "if the served model does not use priority scheduling."))
825
+
826
+ def to_pooling_params(self):
827
+ return PoolingParams(additional_data=self.additional_data)
828
+
829
+
830
class CompletionLogProbs(OpenAIBaseModel):
831
text_offset: List[int] = Field(default_factory=list)
832
token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -876,6 +897,21 @@ class EmbeddingResponse(OpenAIBaseModel):
897
usage: UsageInfo
898
899
900
+class ScoreResponseData(OpenAIBaseModel):
901
+ index: int
902
+ object: str = "score"
903
+ score: Union[List[float], str]
904
+
905
+
906
+class ScoreResponse(OpenAIBaseModel):
907
+ id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
908
+ object: str = "list"
909
+ created: int = Field(default_factory=lambda: int(time.time()))
910
+ model: str
911
+ data: List[ScoreResponseData]
912
+ usage: UsageInfo
913
+
914
+
915
class FunctionCall(OpenAIBaseModel):
916
name: str
917
arguments: str
vllm/entrypoints/openai/serving_score.py
new
+215
@@ -0,0 +1,215 @@
1
+import asyncio
2
+import time
3
+from typing import Any, AsyncGenerator, Dict, List, Optional, Union, cast
4
+
5
+from fastapi import Request
6
+
7
+from vllm.config import ModelConfig
8
+from vllm.engine.protocol import EngineClient
9
+from vllm.entrypoints.logger import RequestLogger
10
+from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
11
+ ScoreResponse, ScoreResponseData,
12
+ UsageInfo)
13
+from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
14
+from vllm.inputs.data import TokensPrompt
15
+from vllm.logger import init_logger
16
+from vllm.outputs import EmbeddingRequestOutput
17
+from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
+from vllm.utils import merge_async_iterators, random_uuid
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+def request_output_to_score_response(
24
+ final_res_batch: List[EmbeddingRequestOutput], request_id: str,
25
+ created_time: int, model_name: str) -> ScoreResponse:
26
+ data: List[ScoreResponseData] = []
27
+ score = None
28
+ num_prompt_tokens = 0
29
+ for idx, final_res in enumerate(final_res_batch):
30
+ if final_res is not None:
31
+ score = final_res.outputs.embedding
32
+ score_data = ScoreResponseData(index=idx, score=score)
33
+ data.append(score_data)
34
+
35
+ usage = UsageInfo(
36
+ prompt_tokens=num_prompt_tokens,
37
+ total_tokens=num_prompt_tokens,
38
+ )
39
+
40
+ return ScoreResponse(
41
+ id=request_id,
42
+ created=created_time,
43
+ model=model_name,
44
+ data=data,
45
+ usage=usage,
46
+ )
47
+
48
+
49
+def make_pairs(text_1: Union[List[str], str], text_2: Union[List[str],
50
+ str]) -> List:
51
+ if isinstance(text_1, (str, dict)):
52
+ # Convert a single prompt to a list.
53
+ text_1 = [text_1]
54
+ text_1 = [t for t in text_1]
55
+
56
+ if isinstance(text_2, (str, dict)):
57
+ # Convert a single prompt to a list.
58
+ text_2 = [text_2]
59
+ text_2 = [t for t in text_2]
60
+ if len(text_1) > 1 and len(text_1) != len(text_2):
61
+ raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
62
+ if len(text_1) == 0:
63
+ raise ValueError("At least one text element must be given")
64
+ if len(text_2) == 0:
65
+ raise ValueError("At least one text_pair element must be given")
66
+
67
+ if len(text_1) == 1:
68
+ text_1 = text_1 * len(text_2)
69
+
70
+ return [(t1, t2) for t1, t2 in zip(text_1, text_2)]
71
+
72
+
73
+class OpenAIServingScores(OpenAIServing):
74
+
75
+ def __init__(
76
+ self,
77
+ engine_client: EngineClient,
78
+ model_config: ModelConfig,
79
+ base_model_paths: List[BaseModelPath],
80
+ *,
81
+ request_logger: Optional[RequestLogger],
82
+ ) -> None:
83
+ super().__init__(engine_client=engine_client,
84
+ model_config=model_config,
85
+ base_model_paths=base_model_paths,
86
+ lora_modules=None,
87
+ prompt_adapters=None,
88
+ request_logger=request_logger)
89
+
90
+ async def create_score(
91
+ self,
92
+ request: ScoreRequest,
93
+ raw_request: Optional[Request] = None,
94
+ ) -> Union[ScoreResponse, ErrorResponse]:
95
+ """
96
+ Score API similar to Sentence Transformers cross encoder
97
+
98
+ See https://sbert.net/docs/package_reference/cross_encoder
99
+ """
100
+ error_check_ret = await self._check_model(request)
101
+ if error_check_ret is not None:
102
+ return error_check_ret
103
+
104
+ model_name = request.model
105
+ request_id = f"score-{random_uuid()}"
106
+ created_time = int(time.monotonic())
107
+ truncate_prompt_tokens = request.truncate_prompt_tokens
108
+
109
+ request_prompts = []
110
+ engine_prompts = []
111
+
112
+ try:
113
+ (
114
+ lora_request,
115
+ prompt_adapter_request,
116
+ ) = self._maybe_get_adapters(request)
117
+
118
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
119
+
120
+ if prompt_adapter_request is not None:
121
+ raise NotImplementedError("Prompt adapter is not supported "
122
+ "for embedding models")
123
+
124
+ if isinstance(tokenizer, MistralTokenizer):
125
+ raise ValueError(
126
+ "MistralTokenizer not supported for cross-encoding")
127
+
128
+ if not self.model_config.is_cross_encoder:
129
+ raise ValueError("Model is not cross encoder.")
130
+
131
+ except ValueError as e:
132
+ logger.exception("Error in preprocessing prompt inputs")
133
+ return self.create_error_response(str(e))
134
+
135
+ # Schedule the request and get the result generator.
136
+ generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
137
+
138
+ input_pairs = make_pairs(request.text_1, request.text_2)
139
+
140
+ for q, t in input_pairs:
141
+ request_prompt = f"{q}{tokenizer.sep_token}{t}"
142
+
143
+ tokenization_kwargs: Dict[str, Any] = {}
144
+ if truncate_prompt_tokens is not None:
145
+ tokenization_kwargs["truncation"] = True
146
+ tokenization_kwargs["max_length"] = truncate_prompt_tokens
147
+
148
+ prompt_inputs = tokenizer(text=q,
149
+ text_pair=t,
150
+ **tokenization_kwargs)
151
+ engine_prompt = TokensPrompt(
152
+ prompt_token_ids=prompt_inputs["input_ids"],
153
+ token_type_ids=prompt_inputs.get("token_type_ids"))
154
+
155
+ request_prompts.append(request_prompt)
156
+ engine_prompts.append(engine_prompt)
157
+
158
+ try:
159
+ pooling_params = request.to_pooling_params()
160
+
161
+ for i, engine_prompt in enumerate(engine_prompts):
162
+ request_id_item = f"{request_id}-{i}"
163
+
164
+ self._log_inputs(request_id_item,
165
+ request_prompts[i],
166
+ params=pooling_params,
167
+ lora_request=lora_request,
168
+ prompt_adapter_request=prompt_adapter_request)
169
+
170
+ trace_headers = (None if raw_request is None else await
171
+ self._get_trace_headers(raw_request.headers))
172
+
173
+ generator = self.engine_client.encode(
174
+ engine_prompt,
175
+ pooling_params,
176
+ request_id_item,
177
+ lora_request=lora_request,
178
+ trace_headers=trace_headers,
179
+ priority=request.priority,
180
+ )
181
+
182
+ generators.append(generator)
183
+ except ValueError as e:
184
+ # TODO: Use a vllm-specific Validation Error
185
+ return self.create_error_response(str(e))
186
+
187
+ result_generator = merge_async_iterators(
188
+ *generators,
189
+ is_cancelled=raw_request.is_disconnected if raw_request else None,
190
+ )
191
+
192
+ num_prompts = len(engine_prompts)
193
+
194
+ # Non-streaming response
195
+ final_res_batch: List[Optional[EmbeddingRequestOutput]]
196
+ final_res_batch = [None] * num_prompts
197
+
198
+ try:
199
+ async for i, res in result_generator:
200
+ final_res_batch[i] = res
201
+
202
+ assert all(final_res is not None for final_res in final_res_batch)
203
+
204
+ final_res_batch_checked = cast(List[EmbeddingRequestOutput],
205
+ final_res_batch)
206
+
207
+ response = request_output_to_score_response(
208
+ final_res_batch_checked, request_id, created_time, model_name)
209
+ except asyncio.CancelledError:
210
+ return self.create_error_response("Client disconnected")
211
+ except ValueError as e:
212
+ # TODO: Use a vllm-specific Validation Error
213
+ return self.create_error_response(str(e))
214
+
215
+ return response
vllm/inputs/data.py
+18
@@ -38,6 +38,9 @@ class TokensPrompt(TypedDict):
38
prompt_token_ids: List[int]
39
"""A list of token IDs to pass to the model."""
40
41
+ token_type_ids: NotRequired[List[int]]
42
+ """A list of token type IDs to pass to the cross encoder model."""
43
+
44
multi_modal_data: NotRequired["MultiModalDataDict"]
45
"""
46
DEPRECATED: Optional multi-modal data to pass to the model,
@@ -133,6 +136,9 @@ class TokenInputs(TypedDict):
136
prompt_token_ids: List[int]
137
"""The token IDs of the prompt."""
138
139
+ token_type_ids: NotRequired[List[int]]
140
+ """The token type IDs of the prompt."""
141
+
142
prompt: NotRequired[str]
143
"""
144
The original prompt text corresponding to the token IDs, if available.
@@ -160,6 +166,7 @@ class TokenInputs(TypedDict):
166
167
def token_inputs(
168
prompt_token_ids: List[int],
169
+ token_type_ids: Optional[List[int]] = None,
170
prompt: Optional[str] = None,
171
multi_modal_data: Optional["MultiModalDataDict"] = None,
172
multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
@@ -170,6 +177,8 @@ def token_inputs(
177
178
if prompt is not None:
179
inputs["prompt"] = prompt
180
+ if token_type_ids is not None:
181
+ inputs["token_type_ids"] = token_type_ids
182
if multi_modal_data is not None:
183
inputs["multi_modal_data"] = multi_modal_data
184
if multi_modal_placeholders is not None:
@@ -234,6 +243,15 @@ class SingletonInputsAdapter:
243
244
assert_never(inputs)
245
246
+ @cached_property
247
+ def token_type_ids(self) -> List[int]:
248
+ inputs = self.inputs
249
+
250
+ if inputs["type"] == "token" or inputs["type"] == "multimodal":
251
+ return inputs.get("token_type_ids", [])
252
+
253
+ assert_never(inputs)
254
+
255
@cached_property
256
def prompt_embeds(self) -> Optional[torch.Tensor]:
257
inputs = self.inputs
vllm/inputs/preprocess.py
+2
@@ -305,6 +305,7 @@ class InputPreprocessor:
305
tokens_content = parsed["content"]
306
307
prompt_token_ids = tokens_content["prompt_token_ids"]
308
+ token_type_ids = tokens_content.get("token_type_ids")
309
multi_modal_data = tokens_content.get("multi_modal_data")
310
mm_processor_kwargs = tokens_content.get("mm_processor_kwargs")
311
@@ -318,6 +319,7 @@ class InputPreprocessor:
319
320
return token_inputs(
321
prompt_token_ids=prompt_token_ids,
322
+ token_type_ids=token_type_ids,
323
multi_modal_data=multi_modal_data,
324
mm_processor_kwargs=mm_processor_kwargs,
325
)
vllm/model_executor/layers/pooler.py
+64
@@ -3,11 +3,14 @@ from typing import List, Optional
3
4
import torch
5
import torch.nn as nn
6
+from transformers import PretrainedConfig
7
8
from vllm.config import PoolerConfig
9
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
10
PoolingTensors)
11
from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
12
+from vllm.transformers_utils.config import (
13
+ get_cross_encoder_activation_function)
14
15
16
class PoolingType(IntEnum):
@@ -152,3 +155,64 @@ class Pooler(nn.Module):
155
]
156
157
return PoolerOutput(outputs=pooled_outputs)
158
+
159
+
160
+class CrossEncodingPooler(nn.Module):
161
+ """A layer that pools specific information from hidden states.
162
+
163
+ This layer does the following:
164
+ 1. Extracts specific tokens or aggregates data based on pooling method.
165
+ 2. Normalizes output if specified.
166
+ 3. Returns structured results as `PoolerOutput`.
167
+
168
+ Attributes:
169
+ pooling_type: The type of pooling to use.
170
+ normalize: Whether to normalize the pooled data.
171
+ """
172
+
173
+ def __init__(
174
+ self,
175
+ config: PretrainedConfig,
176
+ classifier: nn.Module,
177
+ pooler: Optional[nn.Module] = None,
178
+ ):
179
+ super().__init__()
180
+ self.classifier = classifier
181
+ self.pooler = pooler
182
+ self.default_activation_function = \
183
+ get_cross_encoder_activation_function(config)
184
+
185
+ def forward(
186
+ self,
187
+ hidden_states: torch.Tensor,
188
+ pooling_metadata: PoolingMetadata,
189
+ ) -> PoolerOutput:
190
+ """Pools sentence pair scores from the hidden_states."""
191
+
192
+ prompt_lens = PoolingTensors.from_pooling_metadata(
193
+ pooling_metadata, hidden_states.device).prompt_lens
194
+
195
+ offset = 0
196
+ pooled_data_lst = []
197
+ for prompt_len in prompt_lens:
198
+ pooled_data_i = hidden_states[offset:offset + prompt_len]
199
+
200
+ if self.pooler is not None:
201
+ final_shape_tensor = self.pooler(pooled_data_i)
202
+ else:
203
+ final_shape_tensor = self.classifier(pooled_data_i)
204
+
205
+ pooled_data_lst.append(final_shape_tensor)
206
+ offset += prompt_len
207
+
208
+ pooled_output = torch.stack(pooled_data_lst)
209
+
210
+ if self.pooler is not None:
211
+ # apply classifier once on the full batch if possible
212
+ pooled_output = self.classifier(pooled_output)
213
+ logits = self.default_activation_function(pooled_output)
214
+
215
+ pooled_outputs = [
216
+ EmbeddingSequenceGroupOutput(data.tolist()) for data in logits
217
+ ]
218
+ return PoolerOutput(outputs=pooled_outputs)
vllm/model_executor/models/bert.py
+116
-12
@@ -11,14 +11,18 @@ from vllm.model_executor.layers.activation import get_act_fn
11
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
12
QKVParallelLinear,
13
RowParallelLinear)
14
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
14
+from vllm.model_executor.layers.pooler import (CrossEncodingPooler, Pooler,
15
+ PoolingType)
16
from vllm.model_executor.layers.quantization.base_config import (
17
QuantizationConfig)
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
VocabParallelEmbedding)
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
+from vllm.model_executor.models.interfaces import SupportsCrossEncoding
22
from vllm.model_executor.pooling_metadata import PoolingMetadata
23
from vllm.sequence import IntermediateTensors, PoolerOutput
24
+from vllm.transformers_utils.config import (
25
+ get_cross_encoder_activation_function)
26
27
from .utils import maybe_prefix
28
@@ -48,7 +52,9 @@ class BertEmbedding(nn.Module):
52
def forward(
53
self,
54
input_ids: torch.Tensor,
51
- position_ids: Optional[torch.Tensor] = None,
55
+ seq_lens: torch.Tensor,
56
+ position_ids: torch.Tensor,
57
+ token_type_ids: Optional[torch.Tensor] = None,
58
) -> torch.Tensor:
59
input_shape = input_ids.size()
60
@@ -58,17 +64,34 @@ class BertEmbedding(nn.Module):
64
# Position embeddings.
65
position_embeddings = self.position_embeddings(position_ids)
66
61
- # Token type embeddings. (TODO: move off hotpath?)
62
- token_type_embeddings = self.token_type_embeddings(
63
- torch.zeros(input_shape,
64
- dtype=torch.long,
65
- device=inputs_embeds.device))
67
+ if token_type_ids is None:
68
+ token_type_ids = torch.zeros(input_shape,
69
+ dtype=torch.long,
70
+ device=inputs_embeds.device)
71
+
72
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
73
74
embeddings = inputs_embeds + token_type_embeddings + position_embeddings
75
embeddings = self.LayerNorm(embeddings)
76
return embeddings
77
78
79
+class BertPooler(nn.Module):
80
+
81
+ def __init__(self, config: BertConfig):
82
+ super().__init__()
83
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
84
+ self.activation = nn.Tanh()
85
+
86
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
87
+ # We "pool" the model by simply taking the hidden state corresponding
88
+ # to the first token.
89
+ first_token_tensor = hidden_states[0, :]
90
+ pooled_output = self.dense(first_token_tensor)
91
+ pooled_output = self.activation(pooled_output)
92
+ return pooled_output
93
+
94
+
95
class BertEncoder(nn.Module):
96
97
def __init__(self,
@@ -309,7 +332,8 @@ class BertModel(nn.Module):
332
*,
333
vllm_config: VllmConfig,
334
prefix: str = "",
312
- embedding_class: type = BertEmbedding):
335
+ embedding_class: type = BertEmbedding,
336
+ add_pooling_layer: bool = False):
337
super().__init__()
338
config = vllm_config.model_config.hf_config
339
cache_config = vllm_config.cache_config
@@ -319,6 +343,7 @@ class BertModel(nn.Module):
343
cache_config,
344
quant_config,
345
prefix=f"{prefix}.encoder")
346
+ self.pooler = BertPooler(config) if add_pooling_layer else None
347
348
def forward(
349
self,
@@ -328,13 +353,17 @@ class BertModel(nn.Module):
353
attn_metadata: AttentionMetadata,
354
intermediate_tensors: Optional[IntermediateTensors] = None,
355
inputs_embeds: Optional[torch.Tensor] = None,
356
+ token_type_ids: Optional[torch.Tensor] = None,
357
) -> torch.Tensor:
358
if inputs_embeds is not None:
359
hidden_states = inputs_embeds
360
else:
335
- hidden_states = self.embeddings(input_ids=input_ids,
336
- position_ids=position_ids)
337
-
361
+ assert hasattr(attn_metadata, "seq_lens_tensor")
362
+ hidden_states = self.embeddings(
363
+ input_ids=input_ids,
364
+ seq_lens=attn_metadata.seq_lens_tensor,
365
+ position_ids=position_ids,
366
+ token_type_ids=token_type_ids)
367
return self.encoder(hidden_states, kv_caches, attn_metadata)
368
369
def load_weights(self, weights: Iterable[Tuple[str,
@@ -349,7 +378,7 @@ class BertModel(nn.Module):
378
params_dict = dict(self.named_parameters())
379
loaded_params: Set[str] = set()
380
for name, loaded_weight in weights:
352
- if "pooler" in name:
381
+ if self.pooler is None and "pooler" in name:
382
continue
383
for (param_name, weight_name, shard_id) in stacked_params_mapping:
384
if weight_name not in name:
@@ -430,3 +459,78 @@ class BertEmbeddingModel(nn.Module):
459
pooling_type=PoolingType.CLS,
460
normalize=True,
461
softmax=False)
462
+
463
+
464
+class BertForSequenceClassification(nn.Module, SupportsCrossEncoding):
465
+ """A model that uses Bert to provide embedding functionalities.
466
+
467
+ This class encapsulates the BertModel and provides an interface for
468
+ embedding operations and customized pooling functions.
469
+
470
+ Attributes:
471
+ model: An instance of BertModel used for forward operations.
472
+ _pooler: An instance of Pooler used for pooling operations.
473
+ """
474
+
475
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
476
+ super().__init__()
477
+ config = vllm_config.model_config.hf_config
478
+
479
+ self.default_activation_function = \
480
+ get_cross_encoder_activation_function(config)
481
+
482
+ self.num_labels = config.num_labels
483
+ self.bert = BertModel(vllm_config=vllm_config,
484
+ prefix=maybe_prefix(prefix, "bert"),
485
+ embedding_class=BertEmbedding,
486
+ add_pooling_layer=True)
487
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)
488
+ self._pooler = CrossEncodingPooler(config, self.classifier,
489
+ self.bert.pooler)
490
+
491
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
492
+
493
+ self_weights = []
494
+
495
+ def weight_filter():
496
+ for name, weight in weights:
497
+ if name.startswith("bert."):
498
+ yield (name[len("bert."):], weight)
499
+ else:
500
+ self_weights.append((name, weight))
501
+
502
+ self.bert.load_weights(weight_filter())
503
+
504
+ params_dict = dict(self.named_parameters())
505
+
506
+ for name, loaded_weight in self_weights:
507
+ if name.startswith("classifier"):
508
+ param = params_dict[name]
509
+ weight_loader = getattr(param, "weight_loader",
510
+ default_weight_loader)
511
+ weight_loader(param, loaded_weight)
512
+
513
+ def pooler(
514
+ self,
515
+ hidden_states: torch.Tensor,
516
+ pooling_metadata: PoolingMetadata,
517
+ ) -> Optional[PoolerOutput]:
518
+ return self._pooler(hidden_states, pooling_metadata)
519
+
520
+ def forward(
521
+ self,
522
+ input_ids: Optional[torch.Tensor],
523
+ positions: torch.Tensor,
524
+ kv_caches: List[torch.Tensor],
525
+ attn_metadata: AttentionMetadata,
526
+ intermediate_tensors: Optional[IntermediateTensors] = None,
527
+ inputs_embeds: Optional[torch.Tensor] = None,
528
+ token_type_ids: Optional[torch.Tensor] = None,
529
+ ) -> torch.Tensor:
530
+ return self.bert(input_ids=input_ids,
531
+ position_ids=positions,
532
+ kv_caches=kv_caches,
533
+ inputs_embeds=inputs_embeds,
534
+ intermediate_tensors=intermediate_tensors,
535
+ attn_metadata=attn_metadata,
536
+ token_type_ids=token_type_ids)
vllm/model_executor/models/interfaces.py
+36
@@ -7,6 +7,8 @@ from typing_extensions import TypeIs
7
from vllm.logger import init_logger
8
from vllm.utils import supports_kw
9
10
+from .interfaces_base import is_embedding_model
11
+
12
if TYPE_CHECKING:
13
from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
14
from vllm.sequence import IntermediateTensors
@@ -350,3 +352,37 @@ def is_attention_free(
352
return isinstance(model, _IsAttentionFreeType)
353
354
return isinstance(model, IsAttentionFree)
355
+
356
+
357
+@runtime_checkable
358
+class SupportsCrossEncoding(Protocol):
359
+ """The interface required for all models that support cross encoding."""
360
+
361
+ supports_cross_encoding: ClassVar[Literal[True]] = True
362
+
363
+
364
+@overload
365
+def supports_cross_encoding(
366
+ model: Type[object]) -> TypeIs[Type[SupportsCrossEncoding]]:
367
+ ...
368
+
369
+
370
+@overload
371
+def supports_cross_encoding(model: object) -> TypeIs[SupportsCrossEncoding]:
372
+ ...
373
+
374
+
375
+def _supports_cross_encoding(
376
+ model: Union[Type[object], object],
377
+) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
378
+
379
+ if isinstance(model, type):
380
+ return isinstance(model, SupportsCrossEncoding)
381
+
382
+ return isinstance(model, SupportsCrossEncoding)
383
+
384
+
385
+def supports_cross_encoding(
386
+ model: Union[Type[object], object],
387
+) -> Union[TypeIs[Type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
388
+ return is_embedding_model(model) and _supports_cross_encoding(model)
vllm/model_executor/models/registry.py
+21
-2
@@ -21,7 +21,8 @@ from vllm.logger import init_logger
21
from vllm.platforms import current_platform
22
23
from .interfaces import (has_inner_state, is_attention_free,
24
- supports_multimodal, supports_pp)
24
+ supports_cross_encoding, supports_multimodal,
25
+ supports_pp)
26
from .interfaces_base import is_embedding_model, is_text_generation_model
27
28
logger = init_logger(__name__)
@@ -100,6 +101,7 @@ _EMBEDDING_MODELS = {
101
# [Text-only]
102
"BertModel": ("bert", "BertEmbeddingModel"),
103
"RobertaModel": ("roberta", "RobertaEmbeddingModel"),
104
+ "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
105
"XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
106
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
107
"Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
@@ -121,6 +123,14 @@ _EMBEDDING_MODELS = {
123
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration") # noqa: E501,
124
}
125
126
+_CROSS_ENCODER_MODELS = {
127
+ "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
128
+ "RobertaForSequenceClassification": ("roberta",
129
+ "RobertaForSequenceClassification"),
130
+ "XLMRobertaForSequenceClassification": ("roberta",
131
+ "RobertaForSequenceClassification"),
132
+}
133
+
134
_MULTIMODAL_MODELS = {
135
# [Decoder-only]
136
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
@@ -159,6 +169,7 @@ _SPECULATIVE_DECODING_MODELS = {
169
_VLLM_MODELS = {
170
**_TEXT_GENERATION_MODELS,
171
**_EMBEDDING_MODELS,
172
+ **_CROSS_ENCODER_MODELS,
173
**_MULTIMODAL_MODELS,
174
**_SPECULATIVE_DECODING_MODELS,
175
}
@@ -193,6 +204,7 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
204
class _ModelInfo:
205
is_text_generation_model: bool
206
is_embedding_model: bool
207
+ supports_cross_encoding: bool
208
supports_multimodal: bool
209
supports_pp: bool
210
has_inner_state: bool
@@ -203,6 +215,7 @@ class _ModelInfo:
215
return _ModelInfo(
216
is_text_generation_model=is_text_generation_model(model),
217
is_embedding_model=is_embedding_model(model),
218
+ supports_cross_encoding=supports_cross_encoding(model),
219
supports_multimodal=supports_multimodal(model),
220
supports_pp=supports_pp(model),
221
has_inner_state=has_inner_state(model),
@@ -415,6 +428,12 @@ class _ModelRegistry:
428
) -> bool:
429
return self.inspect_model_cls(architectures).is_embedding_model
430
431
+ def is_cross_encoder_model(
432
+ self,
433
+ architectures: Union[str, List[str]],
434
+ ) -> bool:
435
+ return self.inspect_model_cls(architectures).supports_cross_encoding
436
+
437
def is_multimodal_model(
438
self,
439
architectures: Union[str, List[str]],
@@ -489,4 +508,4 @@ def _run() -> None:
508
509
510
if __name__ == "__main__":
492
- _run()
\ No newline at end of file
511
+ _run()
vllm/model_executor/models/roberta.py
+144
-35
@@ -1,4 +1,4 @@
1
-from typing import List, Optional
1
+from typing import Iterable, List, Optional, Tuple
2
3
import torch
4
from torch import nn
@@ -6,10 +6,17 @@ from transformers import RobertaConfig
6
7
from vllm.attention import AttentionMetadata
8
from vllm.config import VllmConfig
9
+from vllm.model_executor.layers.pooler import CrossEncodingPooler
10
from vllm.model_executor.layers.vocab_parallel_embedding import (
11
VocabParallelEmbedding)
12
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
from vllm.model_executor.models.bert import BertEmbeddingModel, BertModel
12
-from vllm.sequence import IntermediateTensors
14
+from vllm.model_executor.models.interfaces import SupportsCrossEncoding
15
+from vllm.model_executor.models.utils import maybe_prefix
16
+from vllm.model_executor.pooling_metadata import PoolingMetadata
17
+from vllm.sequence import IntermediateTensors, PoolerOutput
18
+from vllm.transformers_utils.config import (
19
+ get_cross_encoder_activation_function)
20
21
22
class RobertaEmbedding(nn.Module):
@@ -39,34 +46,93 @@ class RobertaEmbedding(nn.Module):
46
def forward(
47
self,
48
input_ids: torch.Tensor,
42
- position_ids: Optional[torch.Tensor] = None,
49
+ seq_lens: torch.Tensor,
50
+ position_ids: torch.Tensor,
51
+ token_type_ids: Optional[torch.Tensor] = None,
52
) -> torch.Tensor:
53
input_shape = input_ids.size()
45
-
46
- # Input embeddings.
54
inputs_embeds = self.word_embeddings(input_ids)
55
49
- # TODO: figure out if there is a better way
50
- # to make to make position ids start at padding_idx + 1
56
+ # Replace position ids because in RoBERTa models
57
+ # they have to start at padding_idx + 1 and ignore
58
+ # existing padding tokens
59
# References:
60
# - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L133
61
# - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L1669
54
- position_ids += self.padding_idx + 1
62
+ pos_list = []
63
+ token_list = []
64
+ offset = 0
65
+ for seq_len in seq_lens:
66
+ pos_list.append(position_ids[offset:offset + seq_len])
67
+ token_list.append(input_ids[offset:offset + seq_len])
68
+ offset += seq_len
69
+
70
+ new_pos_list = []
71
+ for positions, tokens in zip(pos_list, token_list):
72
+ # Verify assumption that incoming position are
73
+ # always a sequence from 0 to N.
74
+ expected_pos = torch.arange(positions.size()[0],
75
+ dtype=torch.long,
76
+ device=inputs_embeds.device)
77
+ assert torch.equal(positions, expected_pos)
78
+ new_pos_list.append(
79
+ create_position_ids_from_input_ids(tokens, self.padding_idx))
80
+ position_ids = torch.cat(new_pos_list)
81
82
# Position embeddings.
83
position_embeddings = self.position_embeddings(position_ids)
84
+ if token_type_ids is None:
85
+ token_type_ids = torch.zeros(input_shape,
86
+ dtype=torch.long,
87
+ device=inputs_embeds.device)
88
59
- # Token type embeddings. (TODO: move off hotpath?)
60
- token_type_embeddings = self.token_type_embeddings(
61
- torch.zeros(input_shape,
62
- dtype=torch.long,
63
- device=inputs_embeds.device))
64
-
89
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
90
embeddings = inputs_embeds + token_type_embeddings + position_embeddings
91
embeddings = self.LayerNorm(embeddings)
92
return embeddings
93
94
95
+# Adapted from transformers
96
+def create_position_ids_from_input_ids(input_ids,
97
+ padding_idx,
98
+ past_key_values_length=0):
99
+ """
100
+ Replace non-padding symbols with their position numbers.
101
+ Position numbers begin at padding_idx+1. Padding symbols
102
+ are ignored. This is modified from fairseq's `utils.make_positions`.
103
+
104
+ Args:
105
+ x: torch.Tensor x:
106
+
107
+ Returns: torch.Tensor
108
+ """
109
+ # The series of casts and type-conversions here are carefully
110
+ # balanced to both work with ONNX export and XLA.
111
+ mask = input_ids.ne(padding_idx).int()
112
+
113
+ incremental_indices = (torch.cumsum(mask, dim=0).type_as(mask) +
114
+ past_key_values_length) * mask
115
+
116
+ return incremental_indices.long() + padding_idx
117
+
118
+
119
+# Adapted from transformers
120
+class RobertaClassificationHead(nn.Module):
121
+ """Head for sentence-level classification tasks."""
122
+
123
+ def __init__(self, config: RobertaConfig):
124
+ super().__init__()
125
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
126
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
127
+
128
+ def forward(self, features, **kwargs):
129
+ x = features[0, :] # take <s> token (equiv. to [CLS])
130
+ x = self.dense(x)
131
+ x = torch.tanh(x)
132
+ x = self.out_proj(x)
133
+ return x
134
+
135
+
136
class RobertaEmbeddingModel(BertEmbeddingModel):
137
"""A model that uses Roberta to provide embedding functionalities.
138
@@ -85,6 +151,62 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
151
prefix=prefix,
152
embedding_class=RobertaEmbedding)
153
154
+
155
+class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
156
+ """A model that uses Roberta to provide embedding functionalities.
157
+
158
+ This class encapsulates the BertModel and provides an interface for
159
+ embedding operations and customized pooling functions.
160
+
161
+ Attributes:
162
+ roberta: An instance of BertModel used for forward operations.
163
+ _pooler: An instance of Pooler used for pooling operations.
164
+ """
165
+
166
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
167
+ super().__init__()
168
+ config = vllm_config.model_config.hf_config
169
+
170
+ self.default_activation_function = \
171
+ get_cross_encoder_activation_function(config)
172
+
173
+ self.num_labels = config.num_labels
174
+ self.roberta = BertModel(vllm_config=vllm_config,
175
+ prefix=maybe_prefix(prefix, "bert"),
176
+ embedding_class=RobertaEmbedding,
177
+ add_pooling_layer=False)
178
+ self.classifier = RobertaClassificationHead(config)
179
+ self._pooler = CrossEncodingPooler(config, self.classifier)
180
+
181
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
182
+
183
+ self_weights = []
184
+
185
+ def weight_filter():
186
+ for name, weight in weights:
187
+ if name.startswith("roberta."):
188
+ yield (name[len("roberta."):], weight)
189
+ else:
190
+ self_weights.append((name, weight))
191
+
192
+ self.roberta.load_weights(weight_filter())
193
+
194
+ params_dict = dict(self.named_parameters())
195
+
196
+ for name, loaded_weight in self_weights:
197
+ if name.startswith("classifier"):
198
+ param = params_dict[name]
199
+ weight_loader = getattr(param, "weight_loader",
200
+ default_weight_loader)
201
+ weight_loader(param, loaded_weight)
202
+
203
+ def pooler(
204
+ self,
205
+ hidden_states: torch.Tensor,
206
+ pooling_metadata: PoolingMetadata,
207
+ ) -> Optional[PoolerOutput]:
208
+ return self._pooler(hidden_states, pooling_metadata)
209
+
210
def forward(
211
self,
212
input_ids: Optional[torch.Tensor],
@@ -93,25 +215,12 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
215
attn_metadata: AttentionMetadata,
216
intermediate_tensors: Optional[IntermediateTensors] = None,
217
inputs_embeds: Optional[torch.Tensor] = None,
218
+ token_type_ids: Optional[torch.Tensor] = None,
219
) -> torch.Tensor:
97
-
98
- # Verify assumption that position are always a sequence from
99
- # 0 to N. (Actually here we just check 0 and N to simplify).
100
- # This is important to fix the position which are assumed to
101
- # start from padding_idx + 1 instead of 0 in the Roberta models.
102
- assert hasattr(attn_metadata, "seq_lens_tensor")
103
- cumulative = attn_metadata.seq_lens_tensor.cumsum(dim=0)
104
- start_pos = torch.cat(
105
- (torch.tensor([0], device=attn_metadata.seq_lens_tensor.device),
106
- cumulative[:-1]))
107
- assert len(torch.nonzero(positions[start_pos])) == 0
108
- end_pos = cumulative - 1
109
- last_tokens = attn_metadata.seq_lens_tensor - 1
110
- assert len(torch.nonzero(positions[end_pos] - last_tokens)) == 0
111
-
112
- return super().forward(input_ids=input_ids,
113
- positions=positions,
114
- kv_caches=kv_caches,
115
- attn_metadata=attn_metadata,
116
- intermediate_tensors=intermediate_tensors,
117
- inputs_embeds=inputs_embeds)
220
+ return self.roberta(input_ids=input_ids,
221
+ position_ids=positions,
222
+ kv_caches=kv_caches,
223
+ inputs_embeds=inputs_embeds,
224
+ intermediate_tensors=intermediate_tensors,
225
+ attn_metadata=attn_metadata,
226
+ token_type_ids=token_type_ids)
vllm/multimodal/inputs.py
+4
-1
@@ -6,7 +6,7 @@ import numpy as np
6
import torch
7
import torch.types
8
from PIL.Image import Image
9
-from typing_extensions import TypeAlias
9
+from typing_extensions import NotRequired, TypeAlias
10
11
from vllm.utils import JSONTree, is_list_of, json_map_leaves
12
@@ -208,6 +208,9 @@ class MultiModalInputsV2(TypedDict):
208
prompt_token_ids: List[int]
209
"""The processed token IDs which includes placeholder tokens."""
210
211
+ token_type_ids: NotRequired[List[int]]
212
+ """The token type IDs of the prompt."""
213
+
214
mm_kwargs: MultiModalKwargs
215
"""Keyword arguments to be directly passed to the model after batching."""
216
vllm/outputs.py
+44
-1
@@ -60,7 +60,6 @@ class EmbeddingOutput:
60
embedding: The embedding vector, which is a list of floats. The
61
length of vector depends on the model as listed in the embedding guide.
62
"""
63
-
63
embedding: List[float]
64
65
def __repr__(self) -> str:
@@ -363,6 +362,50 @@ class EmbeddingRequestOutput:
362
f"finished={self.finished})")
363
364
365
+@dataclass
366
+class ScoreOutput:
367
+ """The output data of one completion output of a request.
368
+
369
+ Args:
370
+ score: The score, which is a list of floats.
371
+ index: The correspondent text index of the score.
372
+ """
373
+ index: int
374
+ score: List[float]
375
+
376
+ def __repr__(self) -> str:
377
+ return (f"ScoreOutput("
378
+ f"score={self.score}), "
379
+ f"index={self.index})")
380
+
381
+
382
+class ScoreRequestOutput:
383
+ """
384
+ The output data of an score request to the LLM.
385
+
386
+ Args:
387
+ request_id (str): A unique identifier for the score request.
388
+ outputs (score): The embedding results for the given input.
389
+ """
390
+
391
+ def __init__(self, request_id: str, outputs: "ScoreOutput"):
392
+ self.request_id = request_id
393
+ self.outputs = outputs
394
+
395
+ def __repr__(self):
396
+ """
397
+ Returns a string representation of an ScoreRequestOutput instance.
398
+
399
+ The representation includes the request_id and the number of outputs,
400
+ providing a quick overview of the embedding request's results.
401
+
402
+ Returns:
403
+ str: A string representation of the ScoreRequestOutput instance.
404
+ """
405
+ return (f"ScoreRequestOutput(request_id='{self.request_id}', "
406
+ f"outputs={repr(self.outputs)}")
407
+
408
+
409
class RequestOutputFactory:
410
411
@staticmethod
vllm/sequence.py
+9
@@ -449,6 +449,10 @@ class Sequence:
449
def prompt_embeds(self) -> Optional[torch.Tensor]:
450
return self.inputs.prompt_embeds
451
452
+ @property
453
+ def token_type_ids(self) -> List[int]:
454
+ return self.inputs.token_type_ids
455
+
456
@property
457
def multi_modal_data(self) -> "MultiModalDataDict":
458
return self.inputs.multi_modal_data
@@ -687,6 +691,10 @@ class SequenceGroup:
691
return (self.encoder_seq.prompt_token_ids
692
if self.encoder_seq is not None else None)
693
694
+ @property
695
+ def token_type_ids(self) -> Optional[List[int]]:
696
+ return self.first_seq.token_type_ids
697
+
698
@property
699
def multi_modal_data(self) -> MultiModalDataDict:
700
return self.first_seq.multi_modal_data
@@ -909,6 +917,7 @@ class SequenceGroupMetadata(
917
default_factory=lambda: SequenceGroupState())
918
# "MultiModalDataDict" types. We have to use Any due to msgspec
919
# doesn't allow to have union of 2 different dicts.
920
+ token_type_ids: Optional[List[int]] = None
921
multi_modal_data: Optional[Any] = None
922
multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
923
mm_processor_kwargs: Optional[Dict[str, Any]] = None
vllm/transformers_utils/config.py
+15
@@ -9,6 +9,7 @@ from huggingface_hub import (file_exists, hf_hub_download,
9
from huggingface_hub.utils import (EntryNotFoundError, LocalEntryNotFoundError,
10
RepositoryNotFoundError,
11
RevisionNotFoundError)
12
+from torch import nn
13
from transformers import GenerationConfig, PretrainedConfig
14
from transformers.models.auto.image_processing_auto import (
15
get_image_processor_config)
@@ -31,6 +32,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
32
UltravoxConfig)
33
# yapf: enable
34
from vllm.transformers_utils.utils import check_gguf_file
35
+from vllm.utils import resolve_obj_by_qualname
36
37
if VLLM_USE_MODELSCOPE:
38
from modelscope import AutoConfig
@@ -577,3 +579,16 @@ def try_get_generation_config(
579
return GenerationConfig.from_model_config(config)
580
except OSError: # Not found
581
return None
582
+
583
+
584
+def get_cross_encoder_activation_function(config: PretrainedConfig):
585
+ if (hasattr(config, "sbert_ce_default_activation_function")
586
+ and config.sbert_ce_default_activation_function is not None):
587
+
588
+ function_name = config.sbert_ce_default_activation_function
589
+ assert function_name.startswith("torch.nn.modules."), \
590
+ "Loading of activation functions is restricted to " \
591
+ "torch.nn.modules for security reasons"
592
+ return resolve_obj_by_qualname(function_name)()
593
+ else:
594
+ return nn.Sigmoid() if config.num_labels == 1 else nn.Identity()
vllm/worker/cpu_embedding_model_runner.py
+4
@@ -50,6 +50,9 @@ class CPUEmbeddingModelRunner(
50
]
51
52
model_executable = self.model
53
+ cross_enc_kwargs = {}
54
+ if model_input.token_type_ids is not None:
55
+ cross_enc_kwargs["token_type_ids"] = model_input.token_type_ids
56
execute_model_kwargs = {
57
"input_ids":
58
model_input.input_tokens,
@@ -61,6 +64,7 @@ class CPUEmbeddingModelRunner(
64
model_input.attn_metadata,
65
**MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
66
device=self.device),
67
+ **cross_enc_kwargs,
68
"intermediate_tensors":
69
intermediate_tensors,
70
}
vllm/worker/cpu_model_runner.py
+13
@@ -43,6 +43,7 @@ class ModelInputForCPU(ModelRunnerInputBase):
43
"""
44
input_tokens: Optional[torch.Tensor] = None
45
input_positions: Optional[torch.Tensor] = None
46
+ token_type_ids: Optional[torch.Tensor] = None
47
attn_metadata: Optional["AttentionMetadata"] = None
48
multi_modal_kwargs: Optional[BatchedTensorInputs] = None
49
virtual_engine: Optional[int] = None
@@ -54,6 +55,7 @@ class ModelInputForCPU(ModelRunnerInputBase):
55
tensor_dict = {
56
"input_tokens": self.input_tokens,
57
"input_positions": self.input_positions,
58
+ "token_type_ids": self.token_type_ids,
59
"multi_modal_kwargs": self.multi_modal_kwargs,
60
}
61
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
@@ -83,6 +85,7 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
85
tensor_dict = {
86
"input_tokens": self.input_tokens,
87
"input_positions": self.input_positions,
88
+ "token_type_ids": self.token_type_ids,
89
"multi_modal_kwargs": self.multi_modal_kwargs,
90
}
91
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
@@ -112,6 +115,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
115
self.input_tokens: List[int] = []
116
self.input_positions: Optional[
117
List[int]] = [] if not self.use_mrope else None
118
+ self.token_type_ids: Optional[List[int]] = []
119
self.seq_lens: List[int] = []
120
self.query_lens: List[int] = []
121
self.prefill_block_tables: List[List[int]] = []
@@ -165,6 +169,10 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
169
if not input_data.use_mrope else input_data.input_mrope_positions,
170
dtype=torch.long,
171
device="cpu")
172
+ token_type_ids = torch.tensor(input_data.token_type_ids,
173
+ dtype=torch.long,
174
+ device="cpu") \
175
+ if input_data.token_type_ids else None
176
177
# For multi-modal models
178
multi_modal_kwargs = None
@@ -178,6 +186,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
186
return self.model_input_cls(
187
input_tokens=input_tokens,
188
input_positions=input_positions,
189
+ token_type_ids=token_type_ids,
190
seq_lens=input_data.seq_lens,
191
query_lens=input_data.query_lens,
192
attn_metadata=attn_metadata,
@@ -285,6 +294,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
294
tokens = seq_data.get_token_ids()
295
tokens = tokens[context_len:seq_len]
296
token_positions = range(context_len, seq_len)
297
+ token_types = seq_group_metadata.token_type_ids
298
299
# For encoder-only models, the block_table is None,
300
# and there is no need to initialize the slot_mapping.
@@ -301,6 +311,9 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
311
if data.input_positions is not None:
312
data.input_positions.extend(token_positions)
313
314
+ if data.token_type_ids is not None:
315
+ data.token_type_ids.extend(token_types if token_types else [])
316
+
317
# Update fields
318
data.input_tokens.extend(tokens)
319
data.num_prefills += 1
vllm/worker/embedding_model_runner.py
+6
-1
@@ -97,6 +97,10 @@ class EmbeddingModelRunner(
97
model_forward_end = torch.cuda.Event(enable_timing=True)
98
model_forward_start.record()
99
100
+ cross_enc_kwargs = {}
101
+ if model_input.token_types is not None:
102
+ cross_enc_kwargs["token_type_ids"] = model_input.token_types
103
+
104
with set_forward_context(model_input.attn_metadata, self.vllm_config):
105
hidden_or_intermediate_states = model_executable(
106
input_ids=model_input.input_tokens,
@@ -105,7 +109,8 @@ class EmbeddingModelRunner(
109
attn_metadata=model_input.attn_metadata,
110
intermediate_tensors=intermediate_tensors,
111
**MultiModalKwargs.as_kwargs(multi_modal_kwargs,
108
- device=self.device))
112
+ device=self.device),
113
+ **cross_enc_kwargs)
114
115
if (self.observability_config is not None
116
and self.observability_config.collect_model_forward_time):
vllm/worker/model_runner.py
+28
@@ -92,6 +92,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
92
"""
93
input_tokens: Optional[torch.Tensor] = None
94
input_positions: Optional[torch.Tensor] = None
95
+ token_types: Optional[torch.Tensor] = None
96
seq_lens: Optional[List[int]] = None
97
query_lens: Optional[List[int]] = None
98
lora_mapping: Optional["LoRAMapping"] = None
@@ -200,6 +201,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
201
def simple_reinit(self):
202
self.input_tokens[0].clear() # type: ignore
203
self.input_positions[0].clear() # type: ignore
204
+ self.token_types[0].clear() # type: ignore
205
self.mrope_input_positions = None # type: ignore
206
self.seq_lens[0] = 0 # type: ignore
207
self.orig_seq_lens[0] = 0 # type: ignore
@@ -226,6 +228,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
228
# Input tokens and positions.
229
input_tokens: Optional[List[List[int]]] = None,
230
input_positions: Optional[List[List[int]]] = None,
231
+ token_types: Optional[List[List[int]]] = None,
232
mrope_input_positions: Optional[List[List[List[int]]]] = None,
233
234
# The sequence length (may be capped to the sliding window).
@@ -291,6 +294,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
294
for seq_id in range(len(self.seq_ids)):
295
self.input_positions[seq_id].clear()
296
297
+ if token_types:
298
+ self.token_types = token_types
299
+ else:
300
+ for seq_id in range(len(self.seq_ids)):
301
+ self.token_types[seq_id].clear()
302
+
303
self.mrope_input_positions = None
304
305
if seq_lens:
@@ -354,6 +363,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
363
else:
364
self.input_tokens = input_tokens or []
365
self.input_positions = input_positions or []
366
+ self.token_types = token_types or []
367
self.mrope_input_positions = mrope_input_positions or None
368
self.seq_lens = seq_lens or []
369
self.orig_seq_lens = orig_seq_lens or []
@@ -386,6 +396,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
396
397
self.input_tokens = [[] for _ in range(self.n_seqs)]
398
self.input_positions = [[] for _ in range(self.n_seqs)]
399
+ self.token_types = [[] for _ in range(self.n_seqs)]
400
self.mrope_input_positions = None
401
self.seq_lens = [0] * self.n_seqs
402
self.orig_seq_lens = [0] * self.n_seqs
@@ -498,12 +509,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
509
510
# Compute tokens.
511
tokens = seq_data.get_token_ids()[context_len:seq_len]
512
+ token_types = seq_group_metadata.token_type_ids
513
514
inter_data.seq_lens[seq_idx] = seq_len
515
inter_data.orig_seq_lens[seq_idx] = seq_len
516
inter_data.context_lens[seq_idx] = context_len
517
inter_data.input_tokens[seq_idx].extend(tokens)
518
inter_data.input_positions[seq_idx].extend(range(context_len, seq_len))
519
+ inter_data.token_types[seq_idx].extend(
520
+ token_types if token_types else [])
521
inter_data.query_lens[seq_idx] = seq_len - context_len
522
523
if seq_data.mrope_position_delta is not None:
@@ -561,6 +575,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
575
seq_idx][uncomputed_start:]
576
inter_data.input_positions[seq_idx] = inter_data.input_positions[
577
seq_idx][uncomputed_start:]
578
+ inter_data.token_types[seq_idx] = inter_data.token_types[seq_idx][
579
+ uncomputed_start:]
580
context_len = prefix_cache_len
581
582
inter_data.context_lens[seq_idx] = context_len
@@ -575,6 +591,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
591
seq_idx][-1:]
592
inter_data.input_positions[seq_idx] = inter_data.input_positions[
593
seq_idx][-1:]
594
+ inter_data.token_types[seq_idx] = inter_data.token_types[seq_idx][
595
+ -1:]
596
inter_data.query_lens[seq_idx] = 1
597
inter_data.context_lens[seq_idx] = inter_data.seq_lens[seq_idx] - 1
598
@@ -803,9 +821,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
821
"""
822
# Combine and flatten intermediate data.
823
input_tokens = []
824
+ token_types = []
825
for inter_data in self.inter_data_list:
826
for cur_input_tokens in inter_data.input_tokens:
827
input_tokens.extend(cur_input_tokens)
828
+ for cur_token_types in inter_data.token_types:
829
+ token_types.extend(cur_token_types)
830
831
if not input_tokens:
832
# This may happen when all prefill requests hit
@@ -874,6 +895,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
895
input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
896
self.runner.device,
897
self.runner.pin_memory)
898
+
899
+ token_types_tensor = async_tensor_h2d(token_types, torch.long,
900
+ self.runner.device,
901
+ self.runner.pin_memory) \
902
+ if token_types else None
903
+
904
if mrope_input_positions is not None:
905
for idx in range(3):
906
mrope_input_positions[idx].extend(
@@ -952,6 +979,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
979
return self.model_input_cls(
980
input_tokens=input_tokens_tensor,
981
input_positions=input_positions_tensor,
982
+ token_types=token_types_tensor,
983
attn_metadata=attn_metadata,
984
seq_lens=seq_lens,
985
query_lens=query_lens,