@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d697dc01b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix RobertaModel loading (#11940)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Jan 11, 2025 at 15:05 UTC d697dc01b4a25b96c3a1e88d72a058f17a717fd5
3 files changed +67 -12
tests/model_executor/test_model_load_with_params.py
+26 -1
@@ -2,7 +2,7 @@ import os
2
3 import pytest
4
5 -from vllm.model_executor.layers.pooler import PoolingType
5 +from vllm.model_executor.layers.pooler import CLSPool, PoolingType
6 from vllm.model_executor.models.bert import BertEmbeddingModel
7 from vllm.model_executor.models.roberta import RobertaEmbeddingModel
8 from vllm.platforms import current_platform
@@ -92,3 +92,28 @@ def test_roberta_model_loading_with_params(vllm_runner):
92
93 # assert output
94 assert output
95 +
96 +
97 +@pytest.mark.skipif(current_platform.is_rocm(),
98 + reason="Xformers backend is not supported on ROCm.")
99 +def test_facebook_roberta_model_loading_with_params(vllm_runner):
100 + """
101 + Test loading roberta-base model with no lm_head.
102 + """
103 + model_name = "FacebookAI/roberta-base"
104 + with vllm_runner(model_name=model_name,
105 + dtype="float16",
106 + max_model_len=MAX_MODEL_LEN) as model:
107 + output = model.encode("Write a short story about a robot that"
108 + " dreams for the first time.\n")
109 +
110 + model_tokenizer = model.model.llm_engine.tokenizer
111 + assert model_tokenizer.tokenizer_id == model_name
112 +
113 + model = model.model.llm_engine.model_executor\
114 + .driver_worker.model_runner.model
115 + assert not hasattr(model, "lm_head")
116 + assert isinstance(model, RobertaEmbeddingModel)
117 + assert isinstance(model._pooler, CLSPool)
118 +
119 + assert output
tests/models/embedding/language/test_embedding.py
+1
@@ -25,6 +25,7 @@ from ..utils import check_embeddings_close
25 pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
26 pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
27 pytest.param("Alibaba-NLP/gte-Qwen2-7B-instruct"),
28 + pytest.param("sentence-transformers/stsb-roberta-base-v2"),
29 ],
30 )
31 @pytest.mark.parametrize("dtype", ["half"])
vllm/model_executor/models/roberta.py
+40 -11
@@ -1,3 +1,4 @@
1 +import itertools
2 from typing import Iterable, List, Optional, Tuple
3
4 import torch
@@ -20,6 +21,30 @@ from vllm.transformers_utils.config import (
21 from .interfaces import SupportsCrossEncoding
22
23
24 +def roberta_task_weights_filter(
25 + all_weights: Iterable[Tuple[str, torch.Tensor]]
26 +) -> Tuple[Iterable[Tuple[str, torch.Tensor]], Iterable[Tuple[str,
27 + torch.Tensor]]]:
28 + """
29 + Separate task-specific weights that are applied on top
30 + of the encoder-decoder bert base.
31 + To do so, return two generators over the original iterator.
32 + Also, remove the "roberta." prefix to make it loadable
33 + from vanilla BertModel.
34 + """
35 + # Copy of a lazy iterator without in-memory overhead so both
36 + # iterators can be iterated upon independently.
37 + all_weights1, all_weights2 = itertools.tee(all_weights)
38 +
39 + def encoder_decoder_weights():
40 + for name, weight in all_weights1:
41 + if name.startswith("roberta."):
42 + yield (name[len("roberta."):], weight)
43 +
44 + return encoder_decoder_weights(), ((n, w) for n, w in all_weights2
45 + if not n.startswith("roberta."))
46 +
47 +
48 class RobertaEmbedding(nn.Module):
49
50 def __init__(self, config: RobertaConfig):
@@ -152,6 +177,18 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
177 prefix=prefix,
178 embedding_class=RobertaEmbedding)
179
180 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
181 + weights = self.hf_to_vllm_mapper.apply(weights)
182 + # Separate weights in "roberta"-prefixed and all else (not in memory).
183 + # For use with models like FacebookAI/roberta-base.
184 + bert_weights, task_weights = roberta_task_weights_filter(weights)
185 + loaded = self.model.load_weights(bert_weights)
186 + if not len(loaded):
187 + # Fix for models like `sentence-transformers/stsb-roberta-base-v2`
188 + # which use the same architecture, but have no "roberta" prefix.
189 + loaded = self.model.load_weights(task_weights)
190 + assert len(loaded), "Unable to load RobertaEmbeddingModel"
191 +
192
193 class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
194 """A model that uses Roberta to provide embedding functionalities.
@@ -181,20 +218,12 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
218
219 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
220
184 - self_weights = []
185 -
186 - def weight_filter():
187 - for name, weight in weights:
188 - if name.startswith("roberta."):
189 - yield (name[len("roberta."):], weight)
190 - else:
191 - self_weights.append((name, weight))
192 -
193 - self.roberta.load_weights(weight_filter())
221 + bert_weights, task_weights = roberta_task_weights_filter(weights)
222 + self.roberta.load_weights(bert_weights)
223
224 params_dict = dict(self.named_parameters())
225
197 - for name, loaded_weight in self_weights:
226 + for name, loaded_weight in task_weights:
227 if name.startswith("classifier"):
228 param = params_dict[name]
229 weight_loader = getattr(param, "weight_loader",