A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 05:15 UTC
[Bugfix] Fix score layer quantization for sequence classification models - Qwen3 (VL) Reranker (#35849)
Signed-off-by: Hanjun Cho <gkswns0531@gmail.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Hanjun Cho committed
Mar 5, 2026 at 13:57 UTC
f600d5192e287f122b358044f52e17b1d23c06ab
1 file changed
+29
-7
vllm/model_executor/models/adapters.py
+29
-7
@@ -288,15 +288,37 @@ def as_seq_cls_model(cls: _T) -> _T:
288
vllm_config: "VllmConfig",
289
prefix: str = "",
290
) -> "Pooler":
291
- text_config = vllm_config.model_config.hf_config.get_text_config()
291
+ hf_config = vllm_config.model_config.hf_config
292
+ text_config = hf_config.get_text_config()
293
model_config = vllm_config.model_config
293
- quant_config = vllm_config.quant_config
294
+
295
+ # Check if score weights are derived online from LM head
296
+ # (same condition as load_weights branch)
297
+ tokens = getattr(
298
+ hf_config,
299
+ "classifier_from_token",
300
+ getattr(text_config, "classifier_from_token", None),
301
+ )
302
+ method = getattr(
303
+ hf_config,
304
+ "method",
305
+ getattr(text_config, "method", None),
306
+ )
307
+
308
+ # Online conversion: no score weights in checkpoint, don't
309
+ # quantize (small output_dim breaks FP8/Marlin tile alignment).
310
+ # Checkpoint-based: respect the model's quant_config.
311
+ quant_config = (
312
+ None
313
+ if (tokens is not None or method is not None)
314
+ else vllm_config.quant_config
315
+ )
316
317
self.score = ReplicatedLinear(
318
model_config.get_hidden_size(),
319
text_config.num_labels,
320
bias=False,
299
- params_dtype=vllm_config.model_config.head_dtype,
321
+ params_dtype=model_config.head_dtype,
322
quant_config=quant_config,
323
return_bias=False,
324
prefix=maybe_prefix(prefix, "score"),
@@ -452,7 +474,6 @@ def load_weights_using_from_2_way_softmax(
474
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
475
476
model_config = model.vllm_config.model_config
455
- quant_config = model.vllm_config.quant_config
477
hf_config = model.config
478
text_config = hf_config.get_text_config()
479
@@ -469,7 +490,8 @@ def load_weights_using_from_2_way_softmax(
490
using_vlm_head = is_vlm and hasattr(language_model, "score")
491
492
language_model.lm_head = ParallelLMHead(
472
- text_config.vocab_size, text_config.hidden_size, quant_config=quant_config
493
+ text_config.vocab_size,
494
+ text_config.hidden_size,
495
)
496
if text_config.tie_word_embeddings:
497
# embed_tokens is the assumed name for input embeddings. If the model does not
@@ -531,7 +553,6 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
553
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
554
555
model_config = model.vllm_config.model_config
534
- quant_config = model.vllm_config.quant_config
556
text_config = model.config.get_text_config()
557
558
tokens = getattr(text_config, "classifier_from_token", [])
@@ -543,7 +564,8 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
564
using_vlm_head = is_vlm and hasattr(language_model, "score")
565
566
language_model.lm_head = ParallelLMHead(
546
- text_config.vocab_size, text_config.hidden_size, quant_config=quant_config
567
+ text_config.vocab_size,
568
+ text_config.hidden_size,
569
)
570
if text_config.tie_word_embeddings:
571
# embed_tokens is the assumed name for input embeddings. If the model does not