A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 17:45 UTC
[Feature] Add instruction support for score/rerank chat templates (#42412)
Signed-off-by: KrxGu <krishom70@gmail.com>
Krish Gupta committed
May 14, 2026 at 07:11 UTC
70c00163ffa80f5821bc8f87f80bde632bc29015
5 files changed
+182
-12
examples/pooling/score/template/qwen3_reranker.jinja
+1
-1
@@ -1,7 +1,7 @@
1
<|im_start|>system
2
Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>
3
<|im_start|>user
4
-<Instruct>: {{ messages | selectattr("role", "eq", "system") | map(attribute="content") | first | default("Given a web search query, retrieve relevant passages that answer the query") }}
4
+<Instruct>: {{ instruction | default(instruct | default(messages | selectattr("role", "eq", "system") | map(attribute="content") | first | default("Given a web search query, retrieve relevant passages that answer the query", true), true), true) }}
5
<Query>: {{ messages | selectattr("role", "eq", "query") | map(attribute="content") | first }}
6
<Document>: {{ messages | selectattr("role", "eq", "document") | map(attribute="content") | first }}<|im_end|>
7
<|im_start|>assistant
examples/pooling/score/template/qwen3_vl_reranker.jinja
+1
-7
@@ -1,13 +1,7 @@
1
<|im_start|>system
2
Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>
3
<|im_start|>user
4
-<Instruct>: {{
5
- messages
6
- | selectattr("role", "eq", "system")
7
- | map(attribute="content")
8
- | first
9
- | default("Given a search query, retrieve relevant candidates that answer the query.")
10
-}}<Query>:{{
4
+<Instruct>: {{ instruction | default(instruct | default(messages | selectattr("role", "eq", "system") | map(attribute="content") | first | default("Given a search query, retrieve relevant candidates that answer the query.", true), true), true) }}<Query>:{{
5
messages
6
| selectattr("role", "eq", "query")
7
| map(attribute="content")
tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py
+132
@@ -377,3 +377,135 @@ async def test_score_api_queries_list_documents_list(
377
backend,
378
"paired[3]_text_vs_text_plus_image",
379
)
380
+
381
+
382
+INSTRUCTION = (
383
+ "Given a multimodal retrieval query, retrieve candidates that "
384
+ "visually or textually match the requested scene, object, or action."
385
+)
386
+
387
+
388
+@pytest.mark.asyncio
389
+async def test_score_api_instruction_field(
390
+ server: tuple[RemoteOpenAIServer, str],
391
+):
392
+ remote_server, _ = server
393
+
394
+ default_response = requests.post(
395
+ remote_server.url_for("score"),
396
+ json={
397
+ "model": MODEL_NAME,
398
+ "queries": query,
399
+ "documents": document,
400
+ },
401
+ )
402
+ default_response.raise_for_status()
403
+ default_score = ScoreResponse.model_validate(default_response.json())
404
+
405
+ instruction_response = requests.post(
406
+ remote_server.url_for("score"),
407
+ json={
408
+ "model": MODEL_NAME,
409
+ "queries": query,
410
+ "documents": document,
411
+ "instruction": INSTRUCTION,
412
+ },
413
+ )
414
+ instruction_response.raise_for_status()
415
+ instruction_score = ScoreResponse.model_validate(instruction_response.json())
416
+
417
+ assert instruction_score.id is not None
418
+ assert instruction_score.data is not None
419
+ assert len(instruction_score.data) == 1
420
+ assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
421
+
422
+
423
+@pytest.mark.asyncio
424
+async def test_rerank_api_instruction_field(
425
+ server: tuple[RemoteOpenAIServer, str],
426
+):
427
+ remote_server, _ = server
428
+
429
+ doc_list = [
430
+ document,
431
+ {"content": [documents[0]]},
432
+ {"content": [documents[1]]},
433
+ {"content": [documents[0], documents[1]]},
434
+ ]
435
+
436
+ default_response = requests.post(
437
+ remote_server.url_for("rerank"),
438
+ json={
439
+ "model": MODEL_NAME,
440
+ "query": query,
441
+ "documents": doc_list,
442
+ },
443
+ )
444
+ default_response.raise_for_status()
445
+ default_rerank = RerankResponse.model_validate(default_response.json())
446
+
447
+ instruction_response = requests.post(
448
+ remote_server.url_for("rerank"),
449
+ json={
450
+ "model": MODEL_NAME,
451
+ "query": query,
452
+ "documents": doc_list,
453
+ "instruction": INSTRUCTION,
454
+ },
455
+ )
456
+ instruction_response.raise_for_status()
457
+ instruction_rerank = RerankResponse.model_validate(instruction_response.json())
458
+
459
+ assert instruction_rerank.id is not None
460
+ assert instruction_rerank.model is not None
461
+ assert instruction_rerank.usage is not None
462
+ assert len(instruction_rerank.results) == len(default_rerank.results)
463
+ assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
464
+
465
+
466
+@pytest.mark.asyncio
467
+async def test_rerank_api_instruction_field_matches_chat_template_kwargs(
468
+ server: tuple[RemoteOpenAIServer, str],
469
+):
470
+ remote_server, _ = server
471
+
472
+ doc_list = [
473
+ document,
474
+ {"content": [documents[0]]},
475
+ {"content": [documents[1]]},
476
+ {"content": [documents[0], documents[1]]},
477
+ ]
478
+
479
+ field_response = requests.post(
480
+ remote_server.url_for("rerank"),
481
+ json={
482
+ "model": MODEL_NAME,
483
+ "query": query,
484
+ "documents": doc_list,
485
+ "instruction": INSTRUCTION,
486
+ },
487
+ )
488
+ field_response.raise_for_status()
489
+ field_rerank = RerankResponse.model_validate(field_response.json())
490
+
491
+ kwargs_response = requests.post(
492
+ remote_server.url_for("rerank"),
493
+ json={
494
+ "model": MODEL_NAME,
495
+ "query": query,
496
+ "documents": doc_list,
497
+ "chat_template_kwargs": {"instruction": INSTRUCTION},
498
+ },
499
+ )
500
+ kwargs_response.raise_for_status()
501
+ kwargs_rerank = RerankResponse.model_validate(kwargs_response.json())
502
+
503
+ assert kwargs_rerank.usage.prompt_tokens == field_rerank.usage.prompt_tokens
504
+
505
+ field_scores = [
506
+ r.relevance_score for r in sorted(field_rerank.results, key=lambda x: x.index)
507
+ ]
508
+ kwargs_scores = [
509
+ r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index)
510
+ ]
511
+ assert field_scores == pytest.approx(kwargs_scores)
vllm/entrypoints/pooling/scoring/io_processor.py
+17
-2
@@ -157,7 +157,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
157
tok_params,
158
prompt_extras={
159
k: v
160
- for k in ("mm_processor_kwargs", "cache_salt")
160
+ for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
161
if (v := getattr(request, k, None)) is not None
162
},
163
)
@@ -384,7 +384,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
384
max_tokens_per_doc=max_tokens_per_doc,
385
prompt_extras={
386
k: v
387
- for k in ("mm_processor_kwargs", "cache_salt")
387
+ for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
388
if (v := getattr(request, k, None)) is not None
389
},
390
)
@@ -407,6 +407,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
407
pooling_params=ctx.pooling_params
408
)
409
410
+ prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
411
engine_inputs, pooling_params_list = self._pre_process(
412
ctx.prompts,
413
tok_params,
@@ -414,6 +415,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
415
ctx.chat_template,
416
max_tokens_per_query=max_tokens_per_query,
417
max_tokens_per_doc=max_tokens_per_doc,
418
+ prompt_extras=prompt_extras,
419
)
420
ctx.pooling_params = pooling_params_list
421
return engine_inputs
@@ -453,6 +455,9 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
455
chat_template=chat_template,
456
max_tokens_per_query=max_tokens_per_query,
457
max_tokens_per_doc=max_tokens_per_doc,
458
+ chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
459
+ if prompt_extras
460
+ else None,
461
)
462
463
if token_type_ids := engine_prompt.pop("token_type_ids", None):
@@ -477,6 +482,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
482
chat_template: str | None = None,
483
max_tokens_per_query: int = 0,
484
max_tokens_per_doc: int = 0,
485
+ chat_template_kwargs: dict[str, Any] | None = None,
486
):
487
model_config = self.model_config
488
tokenizer = self.tokenizer
@@ -556,6 +562,14 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
562
# If that fails because there is no such template,
563
# fall back to the default implementation.
564
try:
565
+ _safe_kwargs = chat_template_kwargs or {}
566
+ _reserved = {"chat_template", "tools", "tokenize"}
567
+ _unexpected = _reserved & _safe_kwargs.keys()
568
+ if _unexpected:
569
+ raise ValueError(
570
+ "chat_template_kwargs contains reserved keys that "
571
+ f"conflict with fixed scorer arguments: {_unexpected}"
572
+ )
573
full_prompt = safe_apply_chat_template(
574
model_config,
575
tokenizer,
@@ -566,6 +580,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
580
chat_template=chat_template,
581
tools=None,
582
tokenize=False,
583
+ **_safe_kwargs,
584
)
585
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
586
except ChatTemplateResolutionError:
vllm/entrypoints/pooling/scoring/protocol.py
+31
-2
@@ -1,9 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from typing import TypeAlias
4
+from typing import Any, TypeAlias
5
6
-from pydantic import BaseModel, Field
6
+from pydantic import BaseModel, Field, model_validator
7
8
from vllm import PoolingParams
9
from vllm.config import ModelConfig
@@ -35,8 +35,37 @@ class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
35
"applies to the combined query+document)."
36
),
37
)
38
+ instruction: str | None = Field(
39
+ default=None,
40
+ description=(
41
+ "Task instruction prepended to each scored pair via the chat "
42
+ "template. Equivalent to passing "
43
+ "chat_template_kwargs={'instruction': ...}."
44
+ ),
45
+ )
46
+ chat_template_kwargs: dict[str, Any] | None = Field(
47
+ default=None,
48
+ description=(
49
+ "Additional keyword args to pass to the chat template renderer. "
50
+ "Will be accessible by the score/rerank chat template."
51
+ ),
52
+ )
53
# --8<-- [end:scoring-common-params]
54
55
+ @model_validator(mode="after")
56
+ def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
57
+ """Fold the top-level `instruction` field into `chat_template_kwargs`.
58
+
59
+ This allows callers to use either the convenience field or the generic
60
+ dict. Explicit keys inside `chat_template_kwargs` take precedence over
61
+ the top-level `instruction` field.
62
+ """
63
+ if self.instruction is not None:
64
+ merged = dict(self.chat_template_kwargs or {})
65
+ merged.setdefault("instruction", self.instruction)
66
+ self.chat_template_kwargs = merged
67
+ return self
68
+
69
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
70
return self._build_pooling_tok_params(
71
model_config,