A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Frontend] Rerank API (Jina- and Cohere-compatible API) (#12376)
Signed-off-by: Kyle Mistele <kyle@mistele.com>
Kyle Mistele committed
Jan 26, 2025 at 20:58 UTC
0034b09ceb7f578f2d097c7fb8c7042d17367c35
9 files changed
+552
-11
docs/source/serving/openai_compatible_server.md
+92
@@ -50,6 +50,11 @@ In addition, we have the following custom APIs:
50
- Applicable to all [pooling models](../models/pooling_models.md).
51
- [Score API](#score-api) (`/score`)
52
- Only applicable to [cross-encoder models](../models/pooling_models.md) (`--task score`).
53
+- [Re-rank API](#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
54
+ - Implements [Jina AI's v1 re-rank API](https://jina.ai/reranker/)
55
+ - Also compatible with [Cohere's v1 & v2 re-rank APIs](https://docs.cohere.com/v2/reference/rerank)
56
+ - Jina and Cohere's APIs are very similar; Jina's includes extra information in the rerank endpoint's response.
57
+ - Only applicable to [cross-encoder models](../models/pooling_models.md) (`--task score`).
58
59
(chat-template)=
60
@@ -473,3 +478,90 @@ The following extra parameters are supported:
478
:start-after: begin-score-extra-params
479
:end-before: end-score-extra-params
480
```
481
+
482
+(rerank-api)=
483
+
484
+### Re-rank API
485
+
486
+Our Re-rank API applies a cross-encoder model to predict relevant scores between a single query, and
487
+each of a list of documents. Usually, the score for a sentence pair refers to the similarity between two sentences, on
488
+a scale of 0 to 1.
489
+
490
+You can find the documentation for these kind of models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
491
+
492
+The rerank endpoints support popular re-rank models such as `BAAI/bge-reranker-base` and other models supporting the
493
+`score` task. Additionally, `/rerank`, `/v1/rerank`, and `/v2/rerank`
494
+endpoints are compatible with both [Jina AI's re-rank API interface](https://jina.ai/reranker/) and
495
+[Cohere's re-rank API interface](https://docs.cohere.com/v2/reference/rerank) to ensure compatibility with
496
+popular open-source tools.
497
+
498
+Code example: <gh-file:examples/online_serving/jinaai_rerank_client.py>
499
+
500
+#### Example Request
501
+
502
+Note that the `top_n` request parameter is optional and will default to the length of the `documents` field.
503
+Result documents will be sorted by relevance, and the `index` property can be used to determine original order.
504
+
505
+Request:
506
+
507
+```bash
508
+curl -X 'POST' \
509
+ 'http://127.0.0.1:8000/v1/rerank' \
510
+ -H 'accept: application/json' \
511
+ -H 'Content-Type: application/json' \
512
+ -d '{
513
+ "model": "BAAI/bge-reranker-base",
514
+ "query": "What is the capital of France?",
515
+ "documents": [
516
+ "The capital of Brazil is Brasilia.",
517
+ "The capital of France is Paris.",
518
+ "Horses and cows are both animals"
519
+ ]
520
+}'
521
+```
522
+
523
+Response:
524
+
525
+```bash
526
+{
527
+ "id": "rerank-fae51b2b664d4ed38f5969b612edff77",
528
+ "model": "BAAI/bge-reranker-base",
529
+ "usage": {
530
+ "total_tokens": 56
531
+ },
532
+ "results": [
533
+ {
534
+ "index": 1,
535
+ "document": {
536
+ "text": "The capital of France is Paris."
537
+ },
538
+ "relevance_score": 0.99853515625
539
+ },
540
+ {
541
+ "index": 0,
542
+ "document": {
543
+ "text": "The capital of Brazil is Brasilia."
544
+ },
545
+ "relevance_score": 0.0005860328674316406
546
+ }
547
+ ]
548
+}
549
+```
550
+
551
+#### Extra parameters
552
+
553
+The following [pooling parameters](#pooling-params) are supported.
554
+
555
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
556
+:language: python
557
+:start-after: begin-rerank-pooling-params
558
+:end-before: end-rerank-pooling-params
559
+```
560
+
561
+The following extra parameters are supported:
562
+
563
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
564
+:language: python
565
+:start-after: begin-rerank-extra-params
566
+:end-before: end-rerank-extra-params
567
+```
examples/online_serving/cohere_rerank_client.py
new
+32
@@ -0,0 +1,32 @@
1
+"""
2
+Example of using the OpenAI entrypoint's rerank API which is compatible with
3
+the Cohere SDK: https://github.com/cohere-ai/cohere-python
4
+
5
+run: vllm serve BAAI/bge-reranker-base
6
+"""
7
+import cohere
8
+
9
+# cohere v1 client
10
+co = cohere.Client(base_url="http://localhost:8000", api_key="sk-fake-key")
11
+rerank_v1_result = co.rerank(
12
+ model="BAAI/bge-reranker-base",
13
+ query="What is the capital of France?",
14
+ documents=[
15
+ "The capital of France is Paris", "Reranking is fun!",
16
+ "vLLM is an open-source framework for fast AI serving"
17
+ ])
18
+
19
+print(rerank_v1_result)
20
+
21
+# or the v2
22
+co2 = cohere.ClientV2("sk-fake-key", base_url="http://localhost:8000")
23
+
24
+v2_rerank_result = co2.rerank(
25
+ model="BAAI/bge-reranker-base",
26
+ query="What is the capital of France?",
27
+ documents=[
28
+ "The capital of France is Paris", "Reranking is fun!",
29
+ "vLLM is an open-source framework for fast AI serving"
30
+ ])
31
+
32
+print(v2_rerank_result)
examples/online_serving/jinaai_rerank_client.py
new
+33
@@ -0,0 +1,33 @@
1
+"""
2
+Example of using the OpenAI entrypoint's rerank API which is compatible with
3
+Jina and Cohere https://jina.ai/reranker
4
+
5
+run: vllm serve BAAI/bge-reranker-base
6
+"""
7
+import json
8
+
9
+import requests
10
+
11
+url = "http://127.0.0.1:8000/rerank"
12
+
13
+headers = {"accept": "application/json", "Content-Type": "application/json"}
14
+
15
+data = {
16
+ "model":
17
+ "BAAI/bge-reranker-base",
18
+ "query":
19
+ "What is the capital of France?",
20
+ "documents": [
21
+ "The capital of Brazil is Brasilia.",
22
+ "The capital of France is Paris.", "Horses and cows are both animals"
23
+ ]
24
+}
25
+response = requests.post(url, headers=headers, json=data)
26
+
27
+# Check the response
28
+if response.status_code == 200:
29
+ print("Request successful!")
30
+ print(json.dumps(response.json(), indent=2))
31
+else:
32
+ print(f"Request failed with status code: {response.status_code}")
33
+ print(response.text)
tests/entrypoints/openai/test_rerank.py
new
+87
@@ -0,0 +1,87 @@
1
+import pytest
2
+import requests
3
+
4
+from vllm.entrypoints.openai.protocol import RerankResponse
5
+
6
+from ...utils import RemoteOpenAIServer
7
+
8
+MODEL_NAME = "BAAI/bge-reranker-base"
9
+
10
+
11
+@pytest.fixture(scope="module")
12
+def server():
13
+ args = ["--enforce-eager", "--max-model-len", "100"]
14
+
15
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
16
+ yield remote_server
17
+
18
+
19
+@pytest.mark.asyncio
20
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
21
+def test_rerank_texts(server: RemoteOpenAIServer, model_name: str):
22
+ query = "What is the capital of France?"
23
+ documents = [
24
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
25
+ ]
26
+
27
+ rerank_response = requests.post(server.url_for("rerank"),
28
+ json={
29
+ "model": model_name,
30
+ "query": query,
31
+ "documents": documents,
32
+ })
33
+ rerank_response.raise_for_status()
34
+ rerank = RerankResponse.model_validate(rerank_response.json())
35
+
36
+ assert rerank.id is not None
37
+ assert rerank.results is not None
38
+ assert len(rerank.results) == 2
39
+ assert rerank.results[0].relevance_score >= 0.9
40
+ assert rerank.results[1].relevance_score <= 0.01
41
+
42
+
43
+@pytest.mark.asyncio
44
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
45
+def test_top_n(server: RemoteOpenAIServer, model_name: str):
46
+ query = "What is the capital of France?"
47
+ documents = [
48
+ "The capital of Brazil is Brasilia.",
49
+ "The capital of France is Paris.", "Cross-encoder models are neat"
50
+ ]
51
+
52
+ rerank_response = requests.post(server.url_for("rerank"),
53
+ json={
54
+ "model": model_name,
55
+ "query": query,
56
+ "documents": documents,
57
+ "top_n": 2
58
+ })
59
+ rerank_response.raise_for_status()
60
+ rerank = RerankResponse.model_validate(rerank_response.json())
61
+
62
+ assert rerank.id is not None
63
+ assert rerank.results is not None
64
+ assert len(rerank.results) == 2
65
+ assert rerank.results[0].relevance_score >= 0.9
66
+ assert rerank.results[1].relevance_score <= 0.01
67
+
68
+
69
+@pytest.mark.asyncio
70
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
71
+def test_rerank_max_model_len(server: RemoteOpenAIServer, model_name: str):
72
+
73
+ query = "What is the capital of France?" * 100
74
+ documents = [
75
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
76
+ ]
77
+
78
+ rerank_response = requests.post(server.url_for("rerank"),
79
+ json={
80
+ "model": model_name,
81
+ "query": query,
82
+ "documents": documents
83
+ })
84
+ assert rerank_response.status_code == 400
85
+ # Assert just a small fragments of the response
86
+ assert "Please reduce the length of the input." in \
87
+ rerank_response.text
\ No newline at end of file
tests/entrypoints/openai/test_score.py
+1
-6
@@ -10,12 +10,7 @@ MODEL_NAME = "BAAI/bge-reranker-v2-m3"
10
11
@pytest.fixture(scope="module")
12
def server():
13
- args = [
14
- "--enforce-eager",
15
- # Will be used on tests to compare prompt input length
16
- "--max-model-len",
17
- "100"
18
- ]
13
+ args = ["--enforce-eager", "--max-model-len", "100"]
14
15
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
16
yield remote_server
vllm/entrypoints/openai/api_server.py
+50
-1
@@ -56,6 +56,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
56
PoolingChatRequest,
57
PoolingCompletionRequest,
58
PoolingRequest, PoolingResponse,
59
+ RerankRequest, RerankResponse,
60
ScoreRequest, ScoreResponse,
61
TokenizeRequest,
62
TokenizeResponse,
@@ -68,6 +69,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
69
from vllm.entrypoints.openai.serving_models import (BaseModelPath,
70
OpenAIServingModels)
71
from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
72
+from vllm.entrypoints.openai.serving_rerank import JinaAIServingRerank
73
from vllm.entrypoints.openai.serving_score import OpenAIServingScores
74
from vllm.entrypoints.openai.serving_tokenization import (
75
OpenAIServingTokenization)
@@ -306,6 +308,10 @@ def score(request: Request) -> Optional[OpenAIServingScores]:
308
return request.app.state.openai_serving_scores
309
310
311
+def rerank(request: Request) -> Optional[JinaAIServingRerank]:
312
+ return request.app.state.jinaai_serving_reranking
313
+
314
+
315
def tokenization(request: Request) -> OpenAIServingTokenization:
316
return request.app.state.openai_serving_tokenization
317
@@ -502,6 +508,40 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
508
return await create_score(request, raw_request)
509
510
511
+@router.post("/rerank")
512
+@with_cancellation
513
+async def do_rerank(request: RerankRequest, raw_request: Request):
514
+ handler = rerank(raw_request)
515
+ if handler is None:
516
+ return base(raw_request).create_error_response(
517
+ message="The model does not support Rerank (Score) API")
518
+ generator = await handler.do_rerank(request, raw_request)
519
+ if isinstance(generator, ErrorResponse):
520
+ return JSONResponse(content=generator.model_dump(),
521
+ status_code=generator.code)
522
+ elif isinstance(generator, RerankResponse):
523
+ return JSONResponse(content=generator.model_dump())
524
+
525
+ assert_never(generator)
526
+
527
+
528
+@router.post("/v1/rerank")
529
+@with_cancellation
530
+async def do_rerank_v1(request: RerankRequest, raw_request: Request):
531
+ logger.warning(
532
+ "To indicate that the rerank API is not part of the standard OpenAI"
533
+ " API, we have located it at `/rerank`. Please update your client"
534
+ "accordingly. (Note: Conforms to JinaAI rerank API)")
535
+
536
+ return await do_rerank(request, raw_request)
537
+
538
+
539
+@router.post("/v2/rerank")
540
+@with_cancellation
541
+async def do_rerank_v2(request: RerankRequest, raw_request: Request):
542
+ return await do_rerank(request, raw_request)
543
+
544
+
545
TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
546
"generate": {
547
"messages": (ChatCompletionRequest, create_chat_completion),
@@ -512,7 +552,10 @@ TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
552
"default": (EmbeddingCompletionRequest, create_embedding),
553
},
554
"score": {
515
- "default": (ScoreRequest, create_score),
555
+ "default": (RerankRequest, do_rerank)
556
+ },
557
+ "rerank": {
558
+ "default": (RerankRequest, do_rerank)
559
},
560
"reward": {
561
"messages": (PoolingChatRequest, create_pooling),
@@ -759,6 +802,12 @@ async def init_app_state(
802
state.openai_serving_models,
803
request_logger=request_logger
804
) if model_config.task == "score" else None
805
+ state.jinaai_serving_reranking = JinaAIServingRerank(
806
+ engine_client,
807
+ model_config,
808
+ state.openai_serving_models,
809
+ request_logger=request_logger
810
+ ) if model_config.task == "score" else None
811
state.openai_serving_tokenization = OpenAIServingTokenization(
812
engine_client,
813
model_config,
vllm/entrypoints/openai/protocol.py
+46
@@ -1018,6 +1018,52 @@ class ScoreRequest(OpenAIBaseModel):
1018
return PoolingParams(additional_data=self.additional_data)
1019
1020
1021
+class RerankRequest(OpenAIBaseModel):
1022
+ model: str
1023
+ query: str
1024
+ documents: List[str]
1025
+ top_n: int = Field(default_factory=lambda: 0)
1026
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
1027
+
1028
+ # doc: begin-rerank-pooling-params
1029
+ additional_data: Optional[Any] = None
1030
+ # doc: end-rerank-pooling-params
1031
+
1032
+ # doc: begin-rerank-extra-params
1033
+ priority: int = Field(
1034
+ default=0,
1035
+ description=(
1036
+ "The priority of the request (lower means earlier handling; "
1037
+ "default: 0). Any priority other than 0 will raise an error "
1038
+ "if the served model does not use priority scheduling."))
1039
+
1040
+ # doc: end-rerank-extra-params
1041
+
1042
+ def to_pooling_params(self):
1043
+ return PoolingParams(additional_data=self.additional_data)
1044
+
1045
+
1046
+class RerankDocument(BaseModel):
1047
+ text: str
1048
+
1049
+
1050
+class RerankResult(BaseModel):
1051
+ index: int
1052
+ document: RerankDocument
1053
+ relevance_score: float
1054
+
1055
+
1056
+class RerankUsage(BaseModel):
1057
+ total_tokens: int
1058
+
1059
+
1060
+class RerankResponse(OpenAIBaseModel):
1061
+ id: str
1062
+ model: str
1063
+ usage: RerankUsage
1064
+ results: List[RerankResult]
1065
+
1066
+
1067
class CompletionLogProbs(OpenAIBaseModel):
1068
text_offset: List[int] = Field(default_factory=list)
1069
token_logprobs: List[Optional[float]] = Field(default_factory=list)
vllm/entrypoints/openai/serving_engine.py
+5
-4
@@ -26,7 +26,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
26
DetokenizeRequest,
27
EmbeddingChatRequest,
28
EmbeddingCompletionRequest,
29
- ErrorResponse, ScoreRequest,
29
+ ErrorResponse, RerankRequest,
30
+ ScoreRequest,
31
TokenizeChatRequest,
32
TokenizeCompletionRequest)
33
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
@@ -204,9 +205,9 @@ class OpenAIServing:
205
token_num = len(input_ids)
206
207
# Note: EmbeddingRequest and ScoreRequest doesn't have max_tokens
207
- if isinstance(
208
- request,
209
- (EmbeddingChatRequest, EmbeddingCompletionRequest, ScoreRequest)):
208
+ if isinstance(request,
209
+ (EmbeddingChatRequest, EmbeddingCompletionRequest,
210
+ ScoreRequest, RerankRequest)):
211
212
operation = "score" if isinstance(request, ScoreRequest) \
213
else "embedding generation"
vllm/entrypoints/openai/serving_rerank.py
new
+206
@@ -0,0 +1,206 @@
1
+import asyncio
2
+from typing import Any, AsyncGenerator, Dict, List, Optional, Union, cast
3
+
4
+from fastapi import Request
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.engine.protocol import EngineClient
8
+from vllm.entrypoints.logger import RequestLogger
9
+from vllm.entrypoints.openai.protocol import (ErrorResponse, RerankDocument,
10
+ RerankRequest, RerankResponse,
11
+ RerankResult, RerankUsage)
12
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
13
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
14
+from vllm.inputs.data import TokensPrompt
15
+from vllm.logger import init_logger
16
+from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
17
+from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
+from vllm.utils import make_async, merge_async_iterators
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+class JinaAIServingRerank(OpenAIServing):
24
+
25
+ def __init__(
26
+ self,
27
+ engine_client: EngineClient,
28
+ model_config: ModelConfig,
29
+ models: OpenAIServingModels,
30
+ *,
31
+ request_logger: Optional[RequestLogger],
32
+ ) -> None:
33
+ super().__init__(engine_client=engine_client,
34
+ model_config=model_config,
35
+ models=models,
36
+ request_logger=request_logger)
37
+
38
+ async def do_rerank(
39
+ self,
40
+ request: RerankRequest,
41
+ raw_request: Optional[Request] = None
42
+ ) -> Union[RerankResponse, ErrorResponse]:
43
+ """
44
+ Rerank API based on JinaAI's rerank API; implements the same
45
+ API interface. Designed for compatibility with off-the-shelf
46
+ tooling, since this is a common standard for reranking APIs
47
+
48
+ See example client implementations at
49
+ https://github.com/infiniflow/ragflow/blob/main/rag/llm/rerank_model.py
50
+ numerous clients use this standard.
51
+ """
52
+ error_check_ret = await self._check_model(request)
53
+ if error_check_ret is not None:
54
+ return error_check_ret
55
+
56
+ model_name = request.model
57
+ request_id = f"rerank-{self._base_request_id(raw_request)}"
58
+ truncate_prompt_tokens = request.truncate_prompt_tokens
59
+ query = request.query
60
+ documents = request.documents
61
+ request_prompts = []
62
+ engine_prompts = []
63
+ top_n = request.top_n if request.top_n > 0 else len(documents)
64
+
65
+ try:
66
+ (
67
+ lora_request,
68
+ prompt_adapter_request,
69
+ ) = self._maybe_get_adapters(request)
70
+
71
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
72
+
73
+ if prompt_adapter_request is not None:
74
+ raise NotImplementedError("Prompt adapter is not supported "
75
+ "for scoring models")
76
+
77
+ if isinstance(tokenizer, MistralTokenizer):
78
+ raise ValueError(
79
+ "MistralTokenizer not supported for cross-encoding")
80
+
81
+ if not self.model_config.is_cross_encoder:
82
+ raise ValueError("Model is not cross encoder.")
83
+
84
+ if truncate_prompt_tokens is not None and \
85
+ truncate_prompt_tokens > self.max_model_len:
86
+ raise ValueError(
87
+ f"truncate_prompt_tokens value ({truncate_prompt_tokens}) "
88
+ f"is greater than max_model_len ({self.max_model_len})."
89
+ f" Please, select a smaller truncation size.")
90
+ for doc in documents:
91
+ request_prompt = f"{query}{tokenizer.sep_token}{doc}"
92
+ tokenization_kwargs: Dict[str, Any] = {}
93
+ if truncate_prompt_tokens is not None:
94
+ tokenization_kwargs["truncation"] = True
95
+ tokenization_kwargs["max_length"] = truncate_prompt_tokens
96
+
97
+ tokenize_async = make_async(tokenizer.__call__,
98
+ executor=self._tokenizer_executor)
99
+ prompt_inputs = await tokenize_async(text=query,
100
+ text_pair=doc,
101
+ **tokenization_kwargs)
102
+
103
+ input_ids = prompt_inputs["input_ids"]
104
+ text_token_prompt = \
105
+ self._validate_input(request, input_ids, request_prompt)
106
+ engine_prompt = TokensPrompt(
107
+ prompt_token_ids=text_token_prompt["prompt_token_ids"],
108
+ token_type_ids=prompt_inputs.get("token_type_ids"))
109
+
110
+ request_prompts.append(request_prompt)
111
+ engine_prompts.append(engine_prompt)
112
+
113
+ except ValueError as e:
114
+ logger.exception("Error in preprocessing prompt inputs")
115
+ return self.create_error_response(str(e))
116
+
117
+ # Schedule the request and get the result generator.
118
+ generators: List[AsyncGenerator[PoolingRequestOutput, None]] = []
119
+
120
+ try:
121
+ pooling_params = request.to_pooling_params()
122
+
123
+ for i, engine_prompt in enumerate(engine_prompts):
124
+ request_id_item = f"{request_id}-{i}"
125
+
126
+ self._log_inputs(request_id_item,
127
+ request_prompts[i],
128
+ params=pooling_params,
129
+ lora_request=lora_request,
130
+ prompt_adapter_request=prompt_adapter_request)
131
+
132
+ trace_headers = (None if raw_request is None else await
133
+ self._get_trace_headers(raw_request.headers))
134
+
135
+ generator = self.engine_client.encode(
136
+ engine_prompt,
137
+ pooling_params,
138
+ request_id_item,
139
+ lora_request=lora_request,
140
+ trace_headers=trace_headers,
141
+ priority=request.priority,
142
+ )
143
+
144
+ generators.append(generator)
145
+ except ValueError as e:
146
+ # TODO: Use a vllm-specific Validation Error
147
+ return self.create_error_response(str(e))
148
+ result_generator = merge_async_iterators(*generators)
149
+
150
+ num_prompts = len(engine_prompts)
151
+
152
+ # Non-streaming response
153
+ final_res_batch: List[Optional[PoolingRequestOutput]]
154
+ final_res_batch = [None] * num_prompts
155
+
156
+ try:
157
+ async for i, res in result_generator:
158
+ final_res_batch[i] = res
159
+
160
+ assert all(final_res is not None for final_res in final_res_batch)
161
+
162
+ final_res_batch_checked = cast(List[PoolingRequestOutput],
163
+ final_res_batch)
164
+
165
+ response = self.request_output_to_rerank_response(
166
+ final_res_batch_checked, request_id, model_name, documents,
167
+ top_n)
168
+ except asyncio.CancelledError:
169
+ return self.create_error_response("Client disconnected")
170
+ except ValueError as e:
171
+ # TODO: Use a vllm-specific Validation Error
172
+ return self.create_error_response(str(e))
173
+
174
+ return response
175
+
176
+ def request_output_to_rerank_response(
177
+ self, final_res_batch: List[PoolingRequestOutput], request_id: str,
178
+ model_name: str, documents: List[str],
179
+ top_n: int) -> RerankResponse:
180
+ """
181
+ Convert the output of do_rank to a RerankResponse
182
+ """
183
+ results: List[RerankResult] = []
184
+ num_prompt_tokens = 0
185
+ for idx, final_res in enumerate(final_res_batch):
186
+ classify_res = ScoringRequestOutput.from_base(final_res)
187
+
188
+ result = RerankResult(
189
+ index=idx,
190
+ document=RerankDocument(text=documents[idx]),
191
+ relevance_score=classify_res.outputs.score,
192
+ )
193
+ results.append(result)
194
+ prompt_token_ids = final_res.prompt_token_ids
195
+ num_prompt_tokens += len(prompt_token_ids)
196
+
197
+ # sort by relevance, then return the top n if set
198
+ results.sort(key=lambda x: x.relevance_score, reverse=True)
199
+ if top_n < len(documents):
200
+ results = results[:top_n]
201
+
202
+ return RerankResponse(
203
+ id=request_id,
204
+ model=model_name,
205
+ results=results,
206
+ usage=RerankUsage(total_tokens=num_prompt_tokens))