A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Bugfix] Fix Qwen3-VL-Reranker model loading for sequence classification (#32089)
Signed-off-by: rickychen-infinirc <ricky.chen@infinirc.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
RickyChen / 陳昭儒 committed
Jan 11, 2026 at 04:40 UTC
8020a60402597740c7fbd0beba1d5f253983f82f
2 files changed
+18
-12
vllm/model_executor/models/adapters.py
+15
-11
@@ -401,24 +401,23 @@ def load_weights_using_from_2_way_softmax(
401
tokens = cast(list[int], tokens)
402
assert len(tokens) == 2
403
404
- model.lm_head = ParallelLMHead(
404
+ language_model = (
405
+ model.get_language_model() if hasattr(model, "get_language_model") else model
406
+ )
407
+ language_model.lm_head = ParallelLMHead(
408
text_config.vocab_size, text_config.hidden_size, quant_config=quant_config
409
)
410
if text_config.tie_word_embeddings:
411
# embed_tokens is the assumed name for input embeddings. If the model does not
412
# have this attribute, we fall back to get_input_embeddings(), which is used by
413
# the Transformers modeling backend.
411
- text_backbone = (
412
- model.get_language_model().model
413
- if hasattr(model, "get_language_model")
414
- else model.model
415
- )
414
+ text_backbone = language_model.model
415
embed_tokens = (
416
text_backbone.embed_tokens
417
if hasattr(text_backbone, "embed_tokens")
418
else text_backbone.get_input_embeddings()
419
)
421
- model.lm_head = model.lm_head.tie_weights(embed_tokens)
420
+ language_model.lm_head = language_model.lm_head.tie_weights(embed_tokens)
421
422
# ModelForPooling is dynamically defined inside the _create_pooling_model_cls
423
# function, so we need use this hacky method to obtain it.
@@ -438,17 +437,22 @@ def load_weights_using_from_2_way_softmax(
437
438
false_id = tokenizer.convert_tokens_to_ids(tokens[0])
439
true_id = tokenizer.convert_tokens_to_ids(tokens[1])
441
- score_weight = model.lm_head.weight.data[[true_id]].to(
440
+ lm_head_weight = language_model.lm_head.weight
441
+ score_weight = lm_head_weight.data[[true_id]].to(
442
torch.float32
443
- ) - model.lm_head.weight.data[[false_id]].to(torch.float32)
443
+ ) - lm_head_weight.data[[false_id]].to(torch.float32)
444
445
param = model.score.weight
446
weight_loader = getattr(param, "weight_loader", default_weight_loader)
447
weight_loader(param, score_weight)
448
449
- del model.lm_head
449
+ del language_model.lm_head
450
loaded_weights.add("score.weight")
451
- loaded_weights.discard("lm_head.weight")
451
+
452
+ lm_head_name = "lm_head.weight"
453
+ if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
454
+ lm_head_name = hf_to_vllm_mapper._map_name(lm_head_name)
455
+ loaded_weights.discard(lm_head_name)
456
return loaded_weights
457
458
vllm/model_executor/models/config.py
+3
-1
@@ -254,7 +254,9 @@ class Qwen3ForSequenceClassificationConfig(VerifyAndUpdateConfig):
254
"Try loading the original Qwen3 Reranker?, see: "
255
"https://github.com/vllm-project/vllm/tree/main/examples/pooling/score/qwen3_reranker_offline.py"
256
)
257
- model_config.hf_config.method = "from_2_way_softmax"
257
+ text_config = config.get_text_config()
258
+ text_config.method = "from_2_way_softmax"
259
+ text_config.classifier_from_token = tokens
260
261
262
class Qwen3VLForSequenceClassificationConfig(Qwen3ForSequenceClassificationConfig):