| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # ruff: noqa: E501 |
| 4 | """Example of using the rerank API with template. |
| 5 | |
| 6 | This script demonstrates how to interact with a vLLM server running |
| 7 | a reranking model via the REST API. |
| 8 | Before running this script, start the vLLM server with one of the |
| 9 | supported reranking models using the commands below. |
| 10 | |
| 11 | Note: |
| 12 | Some reranking models require special configuration overrides to work correctly |
| 13 | with vLLM's score API. |
| 14 | Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_online.py |
| 15 | Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/convert_model_to_seq_cls.py |
| 16 | |
| 17 | run: |
| 18 | vllm serve BAAI/bge-reranker-v2-gemma --hf_overrides '{"architectures": ["GemmaForSequenceClassification"],"classifier_from_token": ["Yes"],"method": "no_post_processing"}' --chat-template examples/pooling/score/template/bge-reranker-v2-gemma.jinja |
| 19 | vllm serve tomaarsen/Qwen3-Reranker-0.6B-seq-cls --chat-template examples/pooling/score/template/qwen3_reranker.jinja |
| 20 | vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}' --chat-template examples/pooling/score/template/mxbai_rerank_v2.jinja |
| 21 | vllm serve nvidia/llama-nemotron-rerank-1b-v2 --runner pooling --trust-remote-code --chat-template examples/pooling/score/template/nemotron-rerank.jinja |
| 22 | vllm serve Qwen/Qwen3-Reranker-0.6B --runner pooling --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' --chat-template examples/pooling/score/template/qwen3_reranker.jinja |
| 23 | |
| 24 | """ |
| 25 | |
| 26 | import json |
| 27 | |
| 28 | import requests |
| 29 | |
| 30 | # URL of the vLLM server's rerank endpoint |
| 31 | # Default vLLM server runs on localhost port 8000 |
| 32 | url = "http://127.0.0.1:8000/rerank" |
| 33 | |
| 34 | # HTTP headers for the request |
| 35 | headers = {"accept": "application/json", "Content-Type": "application/json"} |
| 36 | |
| 37 | # Example query & documents |
| 38 | query = "how much protein should a female eat?" |
| 39 | documents = [ |
| 40 | "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", |
| 41 | "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.", |
| 42 | "Calorie intake should not fall below 1,200 a day in women or 1,500 a day in men, except under the supervision of a health professional.", |
| 43 | ] |
| 44 | |
| 45 | # Request payload for the rerank API |
| 46 | data = { |
| 47 | "model": "nvidia/llama-nemotron-rerank-1b-v2", # Model to use for reranking |
| 48 | "query": query, # The query to score documents against |
| 49 | "documents": documents, # List of documents to be scored |
| 50 | } |
| 51 | |
| 52 | |
| 53 | def main(): |
| 54 | """Main function to send a rerank request to the vLLM server. |
| 55 | |
| 56 | This function sends a POST request to the /rerank endpoint with |
| 57 | the query and documents, then prints the relevance scores. |
| 58 | """ |
| 59 | # Send POST request to the vLLM server's rerank endpoint |
| 60 | response = requests.post(url, headers=headers, json=data) |
| 61 | |
| 62 | # Check if the request was successful |
| 63 | if response.status_code == 200: |
| 64 | print("Request successful!") |
| 65 | # Pretty print the JSON response containing relevance scores |
| 66 | # The response includes scores for each document's relevance to the query |
| 67 | print(json.dumps(response.json(), indent=2)) |
| 68 | else: |
| 69 | # Handle request failure |
| 70 | print(f"Request failed with status code: {response.status_code}") |
| 71 | print(response.text) |
| 72 | |
| 73 | |
| 74 | if __name__ == "__main__": |
| 75 | main() |