| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # ruff: noqa: E501 |
| 4 | """What is the difference between the official original version and one |
| 5 | that has been converted into a sequence classification model? |
| 6 | |
| 7 | Qwen3-Reranker is a language model that doing reranker by using the |
| 8 | logits of "no" and "yes" tokens. |
| 9 | This requires computing logits for all 151,669 tokens in the vocabulary, |
| 10 | making it inefficient and incompatible with vLLM's score() API. |
| 11 | |
| 12 | A conversion method has been proposed to transform the original model into a |
| 13 | sequence classification model. This converted model: |
| 14 | 1. Is significantly more efficient |
| 15 | 2. Fully supports vLLM's score() API |
| 16 | 3. Simplifies initialization parameters |
| 17 | Reference: https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3 |
| 18 | Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/convert_model_to_seq_cls.py |
| 19 | |
| 20 | For the converted model, initialization would simply be: |
| 21 | vllm serve tomaarsen/Qwen3-Reranker-0.6B-seq-cls --runner pooling --chat-template examples/pooling/score/template/qwen3_reranker.jinja |
| 22 | |
| 23 | This example demonstrates loading the ORIGINAL model with special overrides |
| 24 | to make it compatible with vLLM's score API. |
| 25 | vllm serve Qwen/Qwen3-Reranker-0.6B --runner pooling --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' --chat-template examples/pooling/score/template/qwen3_reranker.jinja |
| 26 | """ |
| 27 | |
| 28 | import json |
| 29 | |
| 30 | import requests |
| 31 | |
| 32 | # URL of the vLLM server's score endpoint |
| 33 | # Default vLLM server runs on localhost port 8000 |
| 34 | url = "http://127.0.0.1:8000/score" |
| 35 | |
| 36 | # HTTP headers for the request |
| 37 | headers = {"accept": "application/json", "Content-Type": "application/json"} |
| 38 | |
| 39 | # Example queries & documents |
| 40 | queries = [ |
| 41 | "What is the capital of China?", |
| 42 | "Explain gravity", |
| 43 | ] |
| 44 | documents = [ |
| 45 | "The capital of China is Beijing.", |
| 46 | "Gravity is a force that attracts two bodies towards each other. It gives weight to physical objects and is responsible for the movement of planets around the sun.", |
| 47 | ] |
| 48 | |
| 49 | # Request payload for the score API |
| 50 | data = { |
| 51 | "model": "Qwen/Qwen3-Reranker-0.6B", |
| 52 | "queries": queries, |
| 53 | "documents": documents, |
| 54 | } |
| 55 | |
| 56 | |
| 57 | def main(): |
| 58 | """Main function to send a score request to the vLLM server. |
| 59 | |
| 60 | This function sends a POST request to the /score endpoint with |
| 61 | the query and documents, then prints the relevance scores. |
| 62 | """ |
| 63 | # Send POST request to the vLLM server's score endpoint |
| 64 | response = requests.post(url, headers=headers, json=data) |
| 65 | |
| 66 | # Check if the request was successful |
| 67 | if response.status_code == 200: |
| 68 | print("Request successful!") |
| 69 | # Pretty print the JSON response containing relevance scores |
| 70 | # The response includes scores for each document's relevance to the query |
| 71 | print(json.dumps(response.json(), indent=2)) |
| 72 | else: |
| 73 | # Handle request failure |
| 74 | print(f"Request failed with status code: {response.status_code}") |
| 75 | print(response.text) |
| 76 | |
| 77 | |
| 78 | if __name__ == "__main__": |
| 79 | main() |