A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
refactor example - qwen3_reranker (#19847)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Jun 24, 2025 at 22:03 UTC
26d34eb67e68dabf2cd9828cbc0b442283dd0c53
1 file changed
+22
-10
examples/offline_inference/qwen3_reranker.py
+22
-10
@@ -22,15 +22,19 @@ model_name = "Qwen/Qwen3-Reranker-0.6B"
22
# If you want to load the official original version, the init parameters are
23
# as follows.
24
25
-model = LLM(
26
- model=model_name,
27
- task="score",
28
- hf_overrides={
29
- "architectures": ["Qwen3ForSequenceClassification"],
30
- "classifier_from_token": ["no", "yes"],
31
- "is_original_qwen3_reranker": True,
32
- },
33
-)
25
+
26
+def get_model() -> LLM:
27
+ """Initializes and returns the LLM model for Qwen3-Reranker."""
28
+ return LLM(
29
+ model=model_name,
30
+ task="score",
31
+ hf_overrides={
32
+ "architectures": ["Qwen3ForSequenceClassification"],
33
+ "classifier_from_token": ["no", "yes"],
34
+ "is_original_qwen3_reranker": True,
35
+ },
36
+ )
37
+
38
39
# Why do we need hf_overrides for the official original version:
40
# vllm converts it to Qwen3ForSequenceClassification when loaded for
@@ -51,7 +55,8 @@ suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
55
query_template = "{prefix}<Instruct>: {instruction}\n<Query>: {query}\n"
56
document_template = "<Document>: {doc}{suffix}"
57
54
-if __name__ == "__main__":
58
+
59
+def main() -> None:
60
instruction = (
61
"Given a web search query, retrieve relevant passages that answer the query"
62
)
@@ -72,6 +77,13 @@ if __name__ == "__main__":
77
]
78
documents = [document_template.format(doc=doc, suffix=suffix) for doc in documents]
79
80
+ model = get_model()
81
outputs = model.score(queries, documents)
82
83
+ print("-" * 30)
84
print([output.outputs.score for output in outputs])
85
+ print("-" * 30)
86
+
87
+
88
+if __name__ == "__main__":
89
+ main()