@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 249824c3bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Refactor `Linear` handling in `TransformersModel` (#12727)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Feb 5, 2025 at 04:31 UTC 249824c3bfef6b9c03dd087569ef1e1072b2a4b0
2 files changed +48 -58
vllm/model_executor/layers/linear.py
+15 -15
@@ -2,7 +2,7 @@
2
3 import itertools
4 from abc import abstractmethod
5 -from typing import Dict, List, Optional, Tuple
5 +from typing import Optional
6
7 import torch
8 import torch.nn.functional as F
@@ -47,8 +47,8 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
47
48
49 def adjust_bitsandbytes_4bit_shard(param: Parameter,
50 - shard_offsets: Dict[str, Tuple[int, int]],
51 - loaded_shard_id: str) -> Tuple[int, int]:
50 + shard_offsets: dict[str, tuple[int, int]],
51 + loaded_shard_id: str) -> tuple[int, int]:
52 """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
53
54 total, _ = shard_offsets["total"]
@@ -90,7 +90,7 @@ class LinearMethodBase(QuantizeMethodBase):
90 @abstractmethod
91 def create_weights(self, layer: torch.nn.Module,
92 input_size_per_partition: int,
93 - output_partition_sizes: List[int], input_size: int,
93 + output_partition_sizes: list[int], input_size: int,
94 output_size: int, params_dtype: torch.dtype,
95 **extra_weight_attrs):
96 """Create weights for a linear layer.
@@ -123,7 +123,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
123
124 def create_weights(self, layer: torch.nn.Module,
125 input_size_per_partition: int,
126 - output_partition_sizes: List[int], input_size: int,
126 + output_partition_sizes: list[int], input_size: int,
127 output_size: int, params_dtype: torch.dtype,
128 **extra_weight_attrs):
129 weight = Parameter(torch.empty(sum(output_partition_sizes),
@@ -179,7 +179,8 @@ class LinearBase(torch.nn.Module):
179 self.quant_method = quant_config.get_quant_method(self,
180 prefix=prefix)
181
182 - def forward(self, x: torch.Tensor) -> torch.Tensor:
182 + def forward(self,
183 + x: torch.Tensor) -> tuple[torch.Tensor, Optional[Parameter]]:
184 raise NotImplementedError
185
186
@@ -240,9 +241,8 @@ class ReplicatedLinear(LinearBase):
241 assert param.size() == loaded_weight.size()
242 param.data.copy_(loaded_weight)
243
243 - def forward(
244 - self, x: torch.Tensor
245 - ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
244 + def forward(self,
245 + x: torch.Tensor) -> tuple[torch.Tensor, Optional[Parameter]]:
246 bias = self.bias if not self.skip_bias_add else None
247 assert self.quant_method is not None
248 output = self.quant_method.apply(self, x, bias)
@@ -288,7 +288,7 @@ class ColumnParallelLinear(LinearBase):
288 skip_bias_add: bool = False,
289 params_dtype: Optional[torch.dtype] = None,
290 quant_config: Optional[QuantizationConfig] = None,
291 - output_sizes: Optional[List[int]] = None,
291 + output_sizes: Optional[list[int]] = None,
292 prefix: str = ""):
293 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
294 quant_config, prefix)
@@ -374,7 +374,7 @@ class ColumnParallelLinear(LinearBase):
374 loaded_weight = loaded_weight.reshape(1)
375 param.load_column_parallel_weight(loaded_weight=loaded_weight)
376
377 - def forward(self, input_):
377 + def forward(self, input_) -> tuple[torch.Tensor, Optional[Parameter]]:
378 bias = self.bias if not self.skip_bias_add else None
379
380 # Matrix multiply.
@@ -422,7 +422,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
422
423 def __init__(self,
424 input_size: int,
425 - output_sizes: List[int],
425 + output_sizes: list[int],
426 bias: bool = True,
427 gather_output: bool = False,
428 skip_bias_add: bool = False,
@@ -500,7 +500,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
500 current_shard_offset = 0
501 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
502 False)
503 - shard_offsets: List[Tuple[int, int, int]] = []
503 + shard_offsets: list[tuple[int, int, int]] = []
504 for i, output_size in enumerate(self.output_sizes):
505 shard_offsets.append((i, current_shard_offset, output_size))
506 current_shard_offset += output_size
@@ -602,7 +602,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
602 """
603
604 current_shard_offset = 0
605 - shard_offsets: List[Tuple[int, int, int]] = []
605 + shard_offsets: list[tuple[int, int, int]] = []
606 for i, output_size in enumerate(self.output_sizes):
607 shard_offsets.append((i, current_shard_offset, output_size))
608 current_shard_offset += output_size
@@ -1124,7 +1124,7 @@ class RowParallelLinear(LinearBase):
1124
1125 param.load_row_parallel_weight(loaded_weight=loaded_weight)
1126
1127 - def forward(self, input_):
1127 + def forward(self, input_) -> tuple[torch.Tensor, Optional[Parameter]]:
1128 if self.input_is_parallel:
1129 input_parallel = input_
1130 else:
vllm/model_executor/models/transformers.py
+33 -43
@@ -1,4 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2 +
3 # Copyright 2024 The vLLM team.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +15,7 @@
15 # limitations under the License.
16 """Wrapper around `transformers` models"""
17 import re
17 -from typing import Iterable, List, Optional, Set, Tuple, Union
18 +from typing import Iterable, Optional, Union
19
20 import torch
21 from torch import nn
@@ -71,23 +72,10 @@ def vllm_flash_attention_forward(
72 ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
73
74
74 -# Linear Layer that is compatible with transformers internal forward
75 -# TODO: This is a temporary solution, we should find a better way to integrate
76 -class HFColumnParallelLinear(ColumnParallelLinear):
77 -
78 - def forward(self, input: torch.Tensor) -> torch.Tensor:
79 - return super().forward(input)[0]
80 -
81 -
82 -class HFRowParallelLinear(RowParallelLinear):
83 -
84 - def forward(self, input: torch.Tensor) -> torch.Tensor:
85 - return super().forward(input)[0]
86 -
87 -
88 -def replace_tp_linear_class(orig_module: nn.Linear,
89 - style: str,
90 - quant_config=None):
75 +def replace_linear_class(
76 + linear: nn.Linear,
77 + style: str,
78 + quant_config=None) -> Union[ColumnParallelLinear, RowParallelLinear]:
79 """
80 In model configurations, we use a neutral type (string) to specify parallel
81 styles, here we use it to translate nn.Linear into vllm-style tp Linear.
@@ -99,26 +87,28 @@ def replace_tp_linear_class(orig_module: nn.Linear,
87 raise ValueError(
88 f"Unsupported parallel style type {type(style)}, expected str")
89
102 - input_size = orig_module.in_features
103 - output_size = orig_module.out_features
104 - bias = orig_module.bias is not None
90 + vllm_linear_cls = {
91 + "colwise": ColumnParallelLinear,
92 + "rowwise": RowParallelLinear,
93 + }.get(style)
94
106 - if style == "colwise":
107 - return HFColumnParallelLinear(
108 - input_size,
109 - output_size,
110 - bias,
111 - )
112 - elif style == "rowwise":
113 - return HFRowParallelLinear(
114 - input_size,
115 - output_size,
116 - bias,
117 - )
118 - # We don't consider colwise_rep since it's used in lm_head
119 - else:
95 + if vllm_linear_cls is None:
96 raise ValueError(f"Unsupported parallel style value: {style}")
97
98 + class HFCompatibleLinear(vllm_linear_cls):
99 + """
100 + Wrapper class that removes `output_bias` from returned output.
101 + """
102 +
103 + def forward(self, input: torch.Tensor) -> torch.Tensor:
104 + return super().forward(input)[0]
105 +
106 + return HFCompatibleLinear(
107 + input_size=linear.in_features,
108 + output_size=linear.out_features,
109 + bias=linear.bias is not None,
110 + )
111 +
112
113 class TransformersModel(nn.Module):
114 embedding_padding_modules = ["lm_head"]
@@ -192,16 +182,16 @@ class TransformersModel(nn.Module):
182 "support it yet!")
183
184 for child_name, child_module in module.named_children():
195 - qual_name = prefix + child_name
185 + qual_name = maybe_prefix(prefix, child_name)
186 for pattern, style in self.config.base_model_tp_plan.items():
187 if re.match(pattern, qual_name) and isinstance(
188 child_module, nn.Linear):
199 - new_module = replace_tp_linear_class(
200 - child_module, style, self.quant_config)
189 + new_module = replace_linear_class(child_module, style,
190 + self.quant_config)
191 setattr(module, child_name, new_module)
192 self.log_replacement(qual_name, child_module, new_module)
193 else:
204 - self.tensor_parallelize(child_module, prefix=f"{qual_name}.")
194 + self.tensor_parallelize(child_module, prefix=qual_name)
195
196 def replace_vocab_embed_class(self, module: nn.Module):
197 # Use native set input embeddings
@@ -219,7 +209,7 @@ class TransformersModel(nn.Module):
209 self,
210 input_ids: torch.Tensor,
211 positions: torch.Tensor,
222 - kv_caches: List[torch.Tensor], # argument not used
212 + kv_caches: list[torch.Tensor], # argument not used
213 attn_metadata: AttentionMetadata,
214 intermediate_tensors: Optional[IntermediateTensors] = None,
215 inputs_embeds: Optional[torch.Tensor] = None,
@@ -249,10 +239,10 @@ class TransformersModel(nn.Module):
239 next_tokens = self.sampler(logits, sampling_metadata)
240 return next_tokens
241
252 - def load_weights(self, weights: Iterable[Tuple[str,
253 - torch.Tensor]]) -> Set[str]:
242 + def load_weights(self, weights: Iterable[tuple[str,
243 + torch.Tensor]]) -> set[str]:
244 params_dict = dict(self.named_parameters())
255 - loaded_params: Set[str] = set()
245 + loaded_params = set[str]()
246 for name, loaded_weight in weights:
247 if name not in params_dict:
248 name = f"{self.model.base_model_prefix}.{name}"