[Model] Enable LoRA support for RobertaForSequenceClassification (#58884)
Signed-off-by: jz-yolo <jz-yolo@users.noreply.github.com> Signed-off-by: Jane Zhu <jane.zhu@slack-corp.com> Co-authored-by: jz-yolo <jz-yolo@users.noreply.github.com>
jz-yolo committed
Sep 27, 2026 at 20:50 UTC
4a190dc518875b3fa5109341a311840e5b023887
2 files changed
+7
-5
docs/models/pooling_models/scoring.md
+2
-2
@@ -50,8 +50,8 @@ The three supported scoring functions are as illustrated in the figure below.
50
| `ModernBertForSequenceClassification` | ModernBERT-based | `Alibaba-NLP/gte-reranker-modernbert-base`, etc. | N/A | | |
51
| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
52
| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
53
-| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
54
-| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
53
+| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | ✅︎ | |
54
+| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | ✅︎ | |
55
| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | N/A | \* | \* |
56
57
<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))
vllm/model_executor/models/roberta.py
+5
-3
@@ -41,7 +41,7 @@ from vllm.model_executor.models.utils import (
41
from vllm.sequence import IntermediateTensors
42
43
from .bert_with_rope import BertWithRope, JinaRobertaModel
44
-from .interfaces import SupportsCrossEncoding
44
+from .interfaces import SupportsCrossEncoding, SupportsLoRA
45
from .interfaces_base import attn_type, default_pooling_type
46
47
@@ -266,7 +266,7 @@ class BgeM3EmbeddingModel(RobertaEmbeddingModel):
266
267
268
@default_pooling_type(seq_pooling_type="CLS")
269
-class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
269
+class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsLoRA):
270
"""A model that uses Roberta to provide embedding functionalities.
271
272
This class encapsulates the BertModel and provides an interface for
@@ -279,6 +279,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
279
"""
280
281
is_pooling_model = True
282
+ packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
283
jina_to_vllm_mapper = WeightsMapper(
284
orig_to_new_substr={
285
"emb_ln": "embeddings.LayerNorm",
@@ -295,7 +296,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
296
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
297
super().__init__()
298
config = vllm_config.model_config.hf_config
298
- self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id
299
+ self.config = config
300
+ self.padding_idx: int = config.pad_token_id
301
302
self.num_labels = config.num_labels
303
self.roberta = BertModel(