1
+# Copyright 2024 The vLLM team.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+"""Wrapper around `transformers` models"""
15
+import re
16
+from typing import Iterable, List, Optional, Set, Tuple, Union
17
+
18
+import torch
19
+from torch import nn
20
+from transformers import AutoModel, PreTrainedModel
21
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
22
+
23
+from vllm.attention import Attention, AttentionMetadata
24
+from vllm.config import VllmConfig
25
+from vllm.distributed import get_tensor_model_parallel_world_size
26
+from vllm.distributed.utils import divide
27
+from vllm.logger import init_logger
28
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
+ RowParallelLinear)
30
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
31
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
32
+from vllm.model_executor.layers.vocab_parallel_embedding import (
33
+ ParallelLMHead, VocabParallelEmbedding)
34
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
+from vllm.model_executor.sampling_metadata import SamplingMetadata
36
+from vllm.sequence import IntermediateTensors
37
+
38
+from .utils import maybe_prefix
39
+
40
+logger = init_logger(__name__)
41
+
42
+
43
+def vllm_flash_attention_forward(
44
+ # Transformers args
45
+ module: torch.nn.Module,
46
+ query: torch.Tensor,
47
+ key: torch.Tensor,
48
+ value: torch.Tensor,
49
+ attention_mask: torch.Tensor,
50
+ # Transformers kwargs
51
+ scaling: float = None,
52
+ # vLLM kwargs
53
+ attn_metadata: AttentionMetadata = None,
54
+ attention_instances: list[Attention] = None,
55
+ **kwargs):
56
+ self_attn = attention_instances[module.layer_idx]
57
+ if scaling is not None:
58
+ self_attn.impl.scale = float(scaling)
59
+ hidden = query.shape[-2]
60
+ query, key, value = (x.transpose(1, 2) for x in (query, key, value))
61
+ query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
62
+ return self_attn.forward(
63
+ query,
64
+ key,
65
+ value,
66
+ kv_cache=None, # argument not used
67
+ attn_metadata=attn_metadata), None
68
+
69
+
70
+ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
71
+
72
+
73
+# Linear Layer that is compatible with transformers internal forward
74
+# TODO: This is a temporary solution, we should find a better way to integrate
75
+class HFColumnParallelLinear(ColumnParallelLinear):
76
+
77
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
78
+ return super().forward(input)[0]
79
+
80
+
81
+class HFRowParallelLinear(RowParallelLinear):
82
+
83
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
84
+ return super().forward(input)[0]
85
+
86
+
87
+def replace_tp_linear_class(orig_module: nn.Linear,
88
+ style: str,
89
+ quant_config=None):
90
+ """
91
+ In model configurations, we use a neutral type (string) to specify parallel
92
+ styles, here we use it to translate nn.Linear into vllm-style tp Linear.
93
+
94
+ Quant config is not supported yet
95
+ """
96
+
97
+ if not isinstance(style, str):
98
+ raise ValueError(
99
+ f"Unsupported parallel style type {type(style)}, expected str")
100
+
101
+ input_size = orig_module.in_features
102
+ output_size = orig_module.out_features
103
+ bias = orig_module.bias is not None
104
+
105
+ if style == "colwise":
106
+ return HFColumnParallelLinear(
107
+ input_size,
108
+ output_size,
109
+ bias,
110
+ )
111
+ elif style == "rowwise":
112
+ return HFRowParallelLinear(
113
+ input_size,
114
+ output_size,
115
+ bias,
116
+ )
117
+ # We don't consider colwise_rep since it's used in lm_head
118
+ else:
119
+ raise ValueError(f"Unsupported parallel style value: {style}")
120
+
121
+
122
+class TransformersModel(nn.Module):
123
+ embedding_padding_modules = ["lm_head"]
124
+ embedding_modules = ["embed_tokens"
125
+ ] # TODO transformers will have a util to get it
126
+
127
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
128
+ super().__init__()
129
+ logger.info("Using Transformers backend.")
130
+
131
+ self.vllm_config = vllm_config
132
+ config = vllm_config.model_config.hf_config
133
+ cache_config = vllm_config.cache_config
134
+ quant_config = vllm_config.quant_config
135
+ self.quant_config = quant_config
136
+ self.config = config
137
+ self.vocab_size = config.vocab_size
138
+ self.unpadded_vocab_size = config.vocab_size
139
+
140
+ self.model: PreTrainedModel = AutoModel.from_config(
141
+ self.config,
142
+ attn_implementation="vllm",
143
+ torch_dtype=vllm_config.model_config.dtype,
144
+ trust_remote_code=vllm_config.model_config.trust_remote_code,
145
+ )
146
+ prefix = self.model.base_model_prefix
147
+
148
+ # MLP modifications
149
+ self.tensor_parallelize(self.model)
150
+
151
+ # Attention modifications (assumes 1 attention op per hidden layer)
152
+ tp_size = get_tensor_model_parallel_world_size()
153
+ self.attention_instances = [
154
+ Attention(
155
+ num_heads=divide(config.num_attention_heads, tp_size),
156
+ head_size=config.head_dim,
157
+ # NOTE: We use Llama scale as default, if it's set by
158
+ # Transformers, it's updated in vllm_flash_attention_forward
159
+ scale=config.head_dim**-0.5,
160
+ num_kv_heads=divide(config.num_key_value_heads, tp_size),
161
+ cache_config=cache_config,
162
+ quant_config=None,
163
+ prefix=f"{i}.attn") for i in range(config.num_hidden_layers)
164
+ ]
165
+
166
+ # Model modifications
167
+ self.replace_vocab_embed_class(self.model)
168
+
169
+ # ForCausalLM modifications
170
+ self.lm_head = ParallelLMHead(config.vocab_size,
171
+ config.hidden_size,
172
+ quant_config=None,
173
+ prefix=maybe_prefix(prefix, "lm_head"))
174
+ if config.tie_word_embeddings:
175
+ self.lm_head.weight = self.model.get_input_embeddings().weight
176
+
177
+ logit_scale = getattr(config, "logit_scale", 1.0)
178
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
179
+ config.vocab_size, logit_scale)
180
+ self.sampler = get_sampler()
181
+
182
+ def log_replacement(self, name: str, old_module: nn.Module,
183
+ new_module: nn.Module):
184
+ logger.debug("%s: %s -> %s", name, old_module, new_module)
185
+
186
+ def tensor_parallelize(self, module: nn.Module, prefix: str = ""):
187
+ if (self.config.base_model_tp_plan is None
188
+ and self.vllm_config.parallel_config.tensor_parallel_size > 1):
189
+ raise ValueError(
190
+ "Trying to run tensor parallelization but the model does not "
191
+ "support it yet!")
192
+
193
+ for child_name, child_module in module.named_children():
194
+ qual_name = prefix + child_name
195
+ for pattern, style in self.config.base_model_tp_plan.items():
196
+ if re.match(pattern, qual_name) and isinstance(
197
+ child_module, nn.Linear):
198
+ new_module = replace_tp_linear_class(
199
+ child_module, style, self.quant_config)
200
+ setattr(module, child_name, new_module)
201
+ self.log_replacement(qual_name, child_module, new_module)
202
+ else:
203
+ self.tensor_parallelize(child_module, prefix=f"{qual_name}.")
204
+
205
+ def replace_vocab_embed_class(self, module: nn.Module):
206
+ # Use native set input embeddings
207
+ new_module = VocabParallelEmbedding(
208
+ self.vocab_size,
209
+ self.config.hidden_size,
210
+ org_num_embeddings=self.config.vocab_size,
211
+ quant_config=None,
212
+ )
213
+ self.log_replacement("input embedding",
214
+ self.model.get_input_embeddings(), new_module)
215
+ self.model.set_input_embeddings(new_module)
216
+
217
+ def forward(
218
+ self,
219
+ input_ids: torch.Tensor,
220
+ positions: torch.Tensor,
221
+ kv_caches: List[torch.Tensor], # argument not used
222
+ attn_metadata: AttentionMetadata,
223
+ intermediate_tensors: Optional[IntermediateTensors] = None,
224
+ inputs_embeds: Optional[torch.Tensor] = None,
225
+ ) -> Union[torch.Tensor, IntermediateTensors]:
226
+ model_output = self.model(
227
+ input_ids[None, ...],
228
+ use_cache=False,
229
+ position_ids=positions[None, ...],
230
+ attn_metadata=attn_metadata,
231
+ intermediate_tensors=intermediate_tensors,
232
+ attention_instances=self.attention_instances,
233
+ return_dict=False)[0][0, ...] # we remove batch dimension for now
234
+ return model_output
235
+
236
+ def compute_logits(
237
+ self,
238
+ hidden_states: torch.Tensor,
239
+ sampling_metadata: SamplingMetadata,
240
+ ) -> Optional[torch.Tensor]:
241
+ logits = self.logits_processor(self.lm_head, hidden_states,
242
+ sampling_metadata)
243
+ return logits
244
+
245
+ def sample(self, logits: torch.Tensor,
246
+ sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:
247
+
248
+ next_tokens = self.sampler(logits, sampling_metadata)
249
+ return next_tokens
250
+
251
+ def load_weights(self, weights: Iterable[Tuple[str,
252
+ torch.Tensor]]) -> Set[str]:
253
+ params_dict = dict(self.named_parameters())
254
+ loaded_params: Set[str] = set()
255
+ for name, loaded_weight in weights:
256
+ if name not in params_dict:
257
+ name = f"{self.model.base_model_prefix}.{name}"
258
+ if name in params_dict:
259
+ param = params_dict[name]
260
+ weight_loader = getattr(param, "weight_loader",
261
+ default_weight_loader)
262
+ weight_loader(param, loaded_weight)
263
+ loaded_params.add(name)
264
+ return loaded_params