A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Refactor `Linear` handling in `TransformersModel` (#12727)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Feb 5, 2025 at 04:31 UTC
249824c3bfef6b9c03dd087569ef1e1072b2a4b0
2 files changed
+48
-58
vllm/model_executor/layers/linear.py
+15
-15
@@ -2,7 +2,7 @@
2
3
import itertools
4
from abc import abstractmethod
5
-from typing import Dict, List, Optional, Tuple
5
+from typing import Optional
6
7
import torch
8
import torch.nn.functional as F
@@ -47,8 +47,8 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
47
48
49
def adjust_bitsandbytes_4bit_shard(param: Parameter,
50
- shard_offsets: Dict[str, Tuple[int, int]],
51
- loaded_shard_id: str) -> Tuple[int, int]:
50
+ shard_offsets: dict[str, tuple[int, int]],
51
+ loaded_shard_id: str) -> tuple[int, int]:
52
"""Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
53
54
total, _ = shard_offsets["total"]
@@ -90,7 +90,7 @@ class LinearMethodBase(QuantizeMethodBase):
90
@abstractmethod
91
def create_weights(self, layer: torch.nn.Module,
92
input_size_per_partition: int,
93
- output_partition_sizes: List[int], input_size: int,
93
+ output_partition_sizes: list[int], input_size: int,
94
output_size: int, params_dtype: torch.dtype,
95
**extra_weight_attrs):
96
"""Create weights for a linear layer.
@@ -123,7 +123,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
123
124
def create_weights(self, layer: torch.nn.Module,
125
input_size_per_partition: int,
126
- output_partition_sizes: List[int], input_size: int,
126
+ output_partition_sizes: list[int], input_size: int,
127
output_size: int, params_dtype: torch.dtype,
128
**extra_weight_attrs):
129
weight = Parameter(torch.empty(sum(output_partition_sizes),
@@ -179,7 +179,8 @@ class LinearBase(torch.nn.Module):
179
self.quant_method = quant_config.get_quant_method(self,
180
prefix=prefix)
181
182
- def forward(self, x: torch.Tensor) -> torch.Tensor:
182
+ def forward(self,
183
+ x: torch.Tensor) -> tuple[torch.Tensor, Optional[Parameter]]:
184
raise NotImplementedError
185
186
@@ -240,9 +241,8 @@ class ReplicatedLinear(LinearBase):
241
assert param.size() == loaded_weight.size()
242
param.data.copy_(loaded_weight)
243
243
- def forward(
244
- self, x: torch.Tensor
245
- ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
244
+ def forward(self,
245
+ x: torch.Tensor) -> tuple[torch.Tensor, Optional[Parameter]]:
246
bias = self.bias if not self.skip_bias_add else None
247
assert self.quant_method is not None
248
output = self.quant_method.apply(self, x, bias)
@@ -288,7 +288,7 @@ class ColumnParallelLinear(LinearBase):
288
skip_bias_add: bool = False,
289
params_dtype: Optional[torch.dtype] = None,
290
quant_config: Optional[QuantizationConfig] = None,
291
- output_sizes: Optional[List[int]] = None,
291
+ output_sizes: Optional[list[int]] = None,
292
prefix: str = ""):
293
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
294
quant_config, prefix)
@@ -374,7 +374,7 @@ class ColumnParallelLinear(LinearBase):
374
loaded_weight = loaded_weight.reshape(1)
375
param.load_column_parallel_weight(loaded_weight=loaded_weight)
376
377
- def forward(self, input_):
377
+ def forward(self, input_) -> tuple[torch.Tensor, Optional[Parameter]]:
378
bias = self.bias if not self.skip_bias_add else None
379
380
# Matrix multiply.
@@ -422,7 +422,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
422
423
def __init__(self,
424
input_size: int,
425
- output_sizes: List[int],
425
+ output_sizes: list[int],
426
bias: bool = True,
427
gather_output: bool = False,
428
skip_bias_add: bool = False,
@@ -500,7 +500,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
500
current_shard_offset = 0
501
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
502
False)
503
- shard_offsets: List[Tuple[int, int, int]] = []
503
+ shard_offsets: list[tuple[int, int, int]] = []
504
for i, output_size in enumerate(self.output_sizes):
505
shard_offsets.append((i, current_shard_offset, output_size))
506
current_shard_offset += output_size
@@ -602,7 +602,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
602
"""
603
604
current_shard_offset = 0
605
- shard_offsets: List[Tuple[int, int, int]] = []
605
+ shard_offsets: list[tuple[int, int, int]] = []
606
for i, output_size in enumerate(self.output_sizes):
607
shard_offsets.append((i, current_shard_offset, output_size))
608
current_shard_offset += output_size
@@ -1124,7 +1124,7 @@ class RowParallelLinear(LinearBase):
1124
1125
param.load_row_parallel_weight(loaded_weight=loaded_weight)
1126
1127
- def forward(self, input_):
1127
+ def forward(self, input_) -> tuple[torch.Tensor, Optional[Parameter]]:
1128
if self.input_is_parallel:
1129
input_parallel = input_
1130
else:
vllm/model_executor/models/transformers.py
+33
-43
@@ -1,4 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+
3
# Copyright 2024 The vLLM team.
4
#
5
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +15,7 @@
15
# limitations under the License.
16
"""Wrapper around `transformers` models"""
17
import re
17
-from typing import Iterable, List, Optional, Set, Tuple, Union
18
+from typing import Iterable, Optional, Union
19
20
import torch
21
from torch import nn
@@ -71,23 +72,10 @@ def vllm_flash_attention_forward(
72
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
73
74
74
-# Linear Layer that is compatible with transformers internal forward
75
-# TODO: This is a temporary solution, we should find a better way to integrate
76
-class HFColumnParallelLinear(ColumnParallelLinear):
77
-
78
- def forward(self, input: torch.Tensor) -> torch.Tensor:
79
- return super().forward(input)[0]
80
-
81
-
82
-class HFRowParallelLinear(RowParallelLinear):
83
-
84
- def forward(self, input: torch.Tensor) -> torch.Tensor:
85
- return super().forward(input)[0]
86
-
87
-
88
-def replace_tp_linear_class(orig_module: nn.Linear,
89
- style: str,
90
- quant_config=None):
75
+def replace_linear_class(
76
+ linear: nn.Linear,
77
+ style: str,
78
+ quant_config=None) -> Union[ColumnParallelLinear, RowParallelLinear]:
79
"""
80
In model configurations, we use a neutral type (string) to specify parallel
81
styles, here we use it to translate nn.Linear into vllm-style tp Linear.
@@ -99,26 +87,28 @@ def replace_tp_linear_class(orig_module: nn.Linear,
87
raise ValueError(
88
f"Unsupported parallel style type {type(style)}, expected str")
89
102
- input_size = orig_module.in_features
103
- output_size = orig_module.out_features
104
- bias = orig_module.bias is not None
90
+ vllm_linear_cls = {
91
+ "colwise": ColumnParallelLinear,
92
+ "rowwise": RowParallelLinear,
93
+ }.get(style)
94
106
- if style == "colwise":
107
- return HFColumnParallelLinear(
108
- input_size,
109
- output_size,
110
- bias,
111
- )
112
- elif style == "rowwise":
113
- return HFRowParallelLinear(
114
- input_size,
115
- output_size,
116
- bias,
117
- )
118
- # We don't consider colwise_rep since it's used in lm_head
119
- else:
95
+ if vllm_linear_cls is None:
96
raise ValueError(f"Unsupported parallel style value: {style}")
97
98
+ class HFCompatibleLinear(vllm_linear_cls):
99
+ """
100
+ Wrapper class that removes `output_bias` from returned output.
101
+ """
102
+
103
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
104
+ return super().forward(input)[0]
105
+
106
+ return HFCompatibleLinear(
107
+ input_size=linear.in_features,
108
+ output_size=linear.out_features,
109
+ bias=linear.bias is not None,
110
+ )
111
+
112
113
class TransformersModel(nn.Module):
114
embedding_padding_modules = ["lm_head"]
@@ -192,16 +182,16 @@ class TransformersModel(nn.Module):
182
"support it yet!")
183
184
for child_name, child_module in module.named_children():
195
- qual_name = prefix + child_name
185
+ qual_name = maybe_prefix(prefix, child_name)
186
for pattern, style in self.config.base_model_tp_plan.items():
187
if re.match(pattern, qual_name) and isinstance(
188
child_module, nn.Linear):
199
- new_module = replace_tp_linear_class(
200
- child_module, style, self.quant_config)
189
+ new_module = replace_linear_class(child_module, style,
190
+ self.quant_config)
191
setattr(module, child_name, new_module)
192
self.log_replacement(qual_name, child_module, new_module)
193
else:
204
- self.tensor_parallelize(child_module, prefix=f"{qual_name}.")
194
+ self.tensor_parallelize(child_module, prefix=qual_name)
195
196
def replace_vocab_embed_class(self, module: nn.Module):
197
# Use native set input embeddings
@@ -219,7 +209,7 @@ class TransformersModel(nn.Module):
209
self,
210
input_ids: torch.Tensor,
211
positions: torch.Tensor,
222
- kv_caches: List[torch.Tensor], # argument not used
212
+ kv_caches: list[torch.Tensor], # argument not used
213
attn_metadata: AttentionMetadata,
214
intermediate_tensors: Optional[IntermediateTensors] = None,
215
inputs_embeds: Optional[torch.Tensor] = None,
@@ -249,10 +239,10 @@ class TransformersModel(nn.Module):
239
next_tokens = self.sampler(logits, sampling_metadata)
240
return next_tokens
241
252
- def load_weights(self, weights: Iterable[Tuple[str,
253
- torch.Tensor]]) -> Set[str]:
242
+ def load_weights(self, weights: Iterable[tuple[str,
243
+ torch.Tensor]]) -> set[str]:
244
params_dict = dict(self.named_parameters())
255
- loaded_params: Set[str] = set()
245
+ loaded_params = set[str]()
246
for name, loaded_weight in weights:
247
if name not in params_dict:
248
name = f"{self.model.base_model_prefix}.{name}"