A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Improve `TransformersModel` UX (#12785)
Harry Mellor committed
Feb 6, 2025 at 06:24 UTC
1a6fcad4c933c89b4060a37d807a7f9e5a680cf3
1 file changed
+32
-21
vllm/model_executor/models/transformers.py
+32
-21
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
"""Wrapper around `transformers` models"""
17
import re
18
-from typing import Iterable, Optional, Union
18
+from typing import Iterable, Literal, Optional, Union
19
20
import torch
21
from torch import nn
@@ -72,15 +72,24 @@ def vllm_flash_attention_forward(
72
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
73
74
75
+def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
76
+ logger.debug("%s: %s -> %s", name, old_module, new_module)
77
+
78
+
79
def replace_linear_class(
80
linear: nn.Linear,
77
- style: str,
81
+ style: Literal["colwise", "rowwise"],
82
quant_config=None) -> Union[ColumnParallelLinear, RowParallelLinear]:
83
"""
80
- In model configurations, we use a neutral type (string) to specify parallel
81
- styles, here we use it to translate nn.Linear into vllm-style tp Linear.
82
-
83
- Quant config is not supported yet
84
+ Replace nn.Linear with one of vLLM's tensor parallel linear classes.
85
+
86
+ `quant_config` is not yet supported.
87
+ Args:
88
+ linear (nn.Linear): `nn.Linear` to be replaced.
89
+ style (str): Tensor parallel style of the new linear, e.g. "colwise".
90
+ quant_config (QuantConfig): Quantization config for the new linear.
91
+ Returns:
92
+ Union[ColumnParallelLinear, RowParallelLinear]: The new linear.
93
"""
94
95
if not isinstance(style, str):
@@ -93,7 +102,10 @@ def replace_linear_class(
102
}.get(style)
103
104
if vllm_linear_cls is None:
96
- raise ValueError(f"Unsupported parallel style value: {style}")
105
+ logger.warning(
106
+ "Unsupported parallel style value: %s. "
107
+ "This layer will not be tensor parallelized.", style)
108
+ return linear
109
110
class HFCompatibleLinear(vllm_linear_cls):
111
"""
@@ -119,25 +131,24 @@ class TransformersModel(nn.Module):
131
super().__init__()
132
logger.info("Using Transformers backend.")
133
122
- self.vllm_config = vllm_config
134
config = vllm_config.model_config.hf_config
135
cache_config = vllm_config.cache_config
136
quant_config = vllm_config.quant_config
126
- self.quant_config = quant_config
137
+
138
self.config = config
139
+ self.quant_config = quant_config
140
self.vocab_size = config.vocab_size
141
self.unpadded_vocab_size = config.vocab_size
142
143
self.model: PreTrainedModel = AutoModel.from_config(
144
self.config,
145
attn_implementation="vllm",
134
- torch_dtype=vllm_config.model_config.dtype,
146
trust_remote_code=vllm_config.model_config.trust_remote_code,
147
)
148
prefix = self.model.base_model_prefix
149
150
# MLP modifications
140
- self.tensor_parallelize(self.model)
151
+ self.apply_base_model_tp_plan(self.model)
152
153
# Attention modifications (assumes 1 attention op per hidden layer)
154
tp_size = get_tensor_model_parallel_world_size()
@@ -170,13 +181,13 @@ class TransformersModel(nn.Module):
181
config.vocab_size, logit_scale)
182
self.sampler = get_sampler()
183
173
- def log_replacement(self, name: str, old_module: nn.Module,
174
- new_module: nn.Module):
175
- logger.debug("%s: %s -> %s", name, old_module, new_module)
176
-
177
- def tensor_parallelize(self, module: nn.Module, prefix: str = ""):
184
+ def apply_base_model_tp_plan(self, module: nn.Module, prefix: str = ""):
185
+ """
186
+ Apply the base model tensor parallelization plan to a module.
187
+ Currently only supports linear layers.
188
+ """
189
if (self.config.base_model_tp_plan is None
179
- and self.vllm_config.parallel_config.tensor_parallel_size > 1):
190
+ and get_tensor_model_parallel_world_size() > 1):
191
raise ValueError(
192
"Trying to run tensor parallelization but the model does not "
193
"support it yet!")
@@ -189,9 +200,9 @@ class TransformersModel(nn.Module):
200
new_module = replace_linear_class(child_module, style,
201
self.quant_config)
202
setattr(module, child_name, new_module)
192
- self.log_replacement(qual_name, child_module, new_module)
203
+ log_replacement(qual_name, child_module, new_module)
204
else:
194
- self.tensor_parallelize(child_module, prefix=qual_name)
205
+ self.apply_base_model_tp_plan(child_module, prefix=qual_name)
206
207
def replace_vocab_embed_class(self, module: nn.Module):
208
# Use native set input embeddings
@@ -201,8 +212,8 @@ class TransformersModel(nn.Module):
212
org_num_embeddings=self.config.vocab_size,
213
quant_config=None,
214
)
204
- self.log_replacement("input embedding",
205
- self.model.get_input_embeddings(), new_module)
215
+ log_replacement("input embedding", self.model.get_input_embeddings(),
216
+ new_module)
217
self.model.set_input_embeddings(new_module)
218
219
def forward(