A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Migrate MistralForCausalLM to LlamaForCausalLM (#2868)
Roy committed
Feb 22, 2024 at 10:25 UTC
344020c926ad19d9d147f5ab6b8929669296edcb
3 files changed
+6
-379
vllm/model_executor/models/__init__.py
+1
-1
@@ -30,7 +30,7 @@ _MODELS = {
30
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
31
# For decapoda-research/llama-*
32
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
33
- "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
33
+ "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
34
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
35
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
36
# transformers's mpt class has lower case
vllm/model_executor/models/llama.py
+5
-1
@@ -92,6 +92,7 @@ class LlamaAttention(nn.Module):
92
max_position_embeddings: int = 8192,
93
linear_method: Optional[LinearMethodBase] = None,
94
bias: bool = False,
95
+ sliding_window: Optional[int] = None,
96
) -> None:
97
super().__init__()
98
self.hidden_size = hidden_size
@@ -141,7 +142,8 @@ class LlamaAttention(nn.Module):
142
self.attn = PagedAttention(self.num_heads,
143
self.head_dim,
144
self.scaling,
144
- num_kv_heads=self.num_kv_heads)
145
+ num_kv_heads=self.num_kv_heads,
146
+ sliding_window=sliding_window)
147
148
def forward(
149
self,
@@ -172,6 +174,7 @@ class LlamaDecoderLayer(nn.Module):
174
rope_scaling = getattr(config, "rope_scaling", None)
175
max_position_embeddings = getattr(config, "max_position_embeddings",
176
8192)
177
+ sliding_window = getattr(config, "sliding_window", None)
178
self.self_attn = LlamaAttention(
179
hidden_size=self.hidden_size,
180
num_heads=config.num_attention_heads,
@@ -182,6 +185,7 @@ class LlamaDecoderLayer(nn.Module):
185
max_position_embeddings=max_position_embeddings,
186
linear_method=linear_method,
187
bias=getattr(config, "bias", False),
188
+ sliding_window=sliding_window,
189
)
190
self.mlp = LlamaMLP(
191
hidden_size=self.hidden_size,
vllm/model_executor/models/mistral.py
deleted
-377
@@ -1,377 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only Mistral model compatible with HuggingFace weights."""
24
-from typing import List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-from transformers import MistralConfig
29
-
30
-from vllm.model_executor.input_metadata import InputMetadata
31
-from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttention
33
-from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
36
- QKVParallelLinear,
37
- RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler
40
-from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
44
-from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.model_executor.weight_utils import (default_weight_loader,
46
- hf_model_weights_iterator)
47
-from vllm.sequence import SamplerOutput
48
-from vllm.config import LoRAConfig
49
-
50
-KVCache = Tuple[torch.Tensor, torch.Tensor]
51
-
52
-
53
-class MistralMLP(nn.Module):
54
-
55
- def __init__(
56
- self,
57
- hidden_size: int,
58
- intermediate_size: int,
59
- hidden_act: str,
60
- linear_method: Optional[LinearMethodBase] = None,
61
- ) -> None:
62
- super().__init__()
63
- self.gate_up_proj = MergedColumnParallelLinear(
64
- hidden_size, [intermediate_size] * 2,
65
- bias=False,
66
- linear_method=linear_method)
67
- self.down_proj = RowParallelLinear(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- linear_method=linear_method)
71
- if hidden_act != "silu":
72
- raise ValueError(f"Unsupported activation: {hidden_act}. "
73
- "Only silu is supported for now.")
74
- self.act_fn = SiluAndMul()
75
-
76
- def forward(self, x):
77
- gate_up, _ = self.gate_up_proj(x)
78
- x = self.act_fn(gate_up)
79
- x, _ = self.down_proj(x)
80
- return x
81
-
82
-
83
-class MistralAttention(nn.Module):
84
-
85
- def __init__(self,
86
- hidden_size: int,
87
- num_heads: int,
88
- num_kv_heads: int,
89
- max_position: int = 4096 * 32,
90
- rope_theta: float = 10000,
91
- linear_method: Optional[LinearMethodBase] = None,
92
- sliding_window: Optional[int] = None) -> None:
93
- super().__init__()
94
- self.hidden_size = hidden_size
95
- tp_size = get_tensor_model_parallel_world_size()
96
- self.total_num_heads = num_heads
97
- assert self.total_num_heads % tp_size == 0
98
- self.num_heads = self.total_num_heads // tp_size
99
- self.total_num_kv_heads = num_kv_heads
100
- if self.total_num_kv_heads >= tp_size:
101
- # Number of KV heads is greater than TP size, so we partition
102
- # the KV heads across multiple tensor parallel GPUs.
103
- assert self.total_num_kv_heads % tp_size == 0
104
- else:
105
- # Number of KV heads is less than TP size, so we replicate
106
- # the KV heads across multiple tensor parallel GPUs.
107
- assert tp_size % self.total_num_kv_heads == 0
108
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109
- self.head_dim = hidden_size // self.total_num_heads
110
- self.q_size = self.num_heads * self.head_dim
111
- self.kv_size = self.num_kv_heads * self.head_dim
112
- self.scaling = self.head_dim**-0.5
113
- self.rope_theta = rope_theta
114
- self.sliding_window = sliding_window
115
-
116
- self.qkv_proj = QKVParallelLinear(
117
- hidden_size,
118
- self.head_dim,
119
- self.total_num_heads,
120
- self.total_num_kv_heads,
121
- bias=False,
122
- linear_method=linear_method,
123
- )
124
- self.o_proj = RowParallelLinear(
125
- self.total_num_heads * self.head_dim,
126
- hidden_size,
127
- bias=False,
128
- linear_method=linear_method,
129
- )
130
-
131
- self.rotary_emb = get_rope(
132
- self.head_dim,
133
- rotary_dim=self.head_dim,
134
- max_position=max_position,
135
- base=self.rope_theta,
136
- )
137
- self.attn = PagedAttention(self.num_heads,
138
- self.head_dim,
139
- self.scaling,
140
- num_kv_heads=self.num_kv_heads,
141
- sliding_window=self.sliding_window)
142
-
143
- def forward(
144
- self,
145
- positions: torch.Tensor,
146
- hidden_states: torch.Tensor,
147
- kv_cache: KVCache,
148
- input_metadata: InputMetadata,
149
- ) -> torch.Tensor:
150
- qkv, _ = self.qkv_proj(hidden_states)
151
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
- q, k = self.rotary_emb(positions, q, k)
153
- k_cache, v_cache = kv_cache
154
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
- output, _ = self.o_proj(attn_output)
156
- return output
157
-
158
-
159
-class MistralDecoderLayer(nn.Module):
160
-
161
- def __init__(
162
- self,
163
- config: MistralConfig,
164
- linear_method: Optional[LinearMethodBase] = None,
165
- ) -> None:
166
- super().__init__()
167
- self.hidden_size = config.hidden_size
168
- # Requires transformers > 4.32.0
169
- rope_theta = getattr(config, "rope_theta", 10000)
170
- self.self_attn = MistralAttention(
171
- hidden_size=self.hidden_size,
172
- num_heads=config.num_attention_heads,
173
- max_position=config.max_position_embeddings,
174
- num_kv_heads=config.num_key_value_heads,
175
- rope_theta=rope_theta,
176
- linear_method=linear_method,
177
- sliding_window=config.sliding_window)
178
- self.mlp = MistralMLP(
179
- hidden_size=self.hidden_size,
180
- intermediate_size=config.intermediate_size,
181
- hidden_act=config.hidden_act,
182
- linear_method=linear_method,
183
- )
184
- self.input_layernorm = RMSNorm(config.hidden_size,
185
- eps=config.rms_norm_eps)
186
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
187
- eps=config.rms_norm_eps)
188
-
189
- def forward(
190
- self,
191
- positions: torch.Tensor,
192
- hidden_states: torch.Tensor,
193
- kv_cache: KVCache,
194
- input_metadata: InputMetadata,
195
- residual: Optional[torch.Tensor],
196
- ) -> Tuple[torch.Tensor, torch.Tensor]:
197
- # Self Attention
198
- if residual is None:
199
- residual = hidden_states
200
- hidden_states = self.input_layernorm(hidden_states)
201
- else:
202
- hidden_states, residual = self.input_layernorm(
203
- hidden_states, residual)
204
- hidden_states = self.self_attn(
205
- positions=positions,
206
- hidden_states=hidden_states,
207
- kv_cache=kv_cache,
208
- input_metadata=input_metadata,
209
- )
210
-
211
- # Fully Connected
212
- hidden_states, residual = self.post_attention_layernorm(
213
- hidden_states, residual)
214
- hidden_states = self.mlp(hidden_states)
215
- return hidden_states, residual
216
-
217
-
218
-class MistralModel(nn.Module):
219
-
220
- def __init__(
221
- self,
222
- config: MistralConfig,
223
- linear_method: Optional[LinearMethodBase] = None,
224
- lora_config: Optional[LoRAConfig] = None,
225
- ) -> None:
226
- super().__init__()
227
- self.config = config
228
- self.padding_idx = config.pad_token_id
229
- lora_vocab = (lora_config.lora_extra_vocab_size *
230
- (lora_config.max_loras or 1)) if lora_config else 0
231
- self.vocab_size = config.vocab_size + lora_vocab
232
- self.org_vocab_size = config.vocab_size
233
-
234
- self.embed_tokens = VocabParallelEmbedding(
235
- self.vocab_size,
236
- config.hidden_size,
237
- org_num_embeddings=config.vocab_size,
238
- )
239
- self.layers = nn.ModuleList([
240
- MistralDecoderLayer(config, linear_method)
241
- for _ in range(config.num_hidden_layers)
242
- ])
243
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
244
-
245
- def forward(
246
- self,
247
- input_ids: torch.Tensor,
248
- positions: torch.Tensor,
249
- kv_caches: List[KVCache],
250
- input_metadata: InputMetadata,
251
- ) -> torch.Tensor:
252
- hidden_states = self.embed_tokens(input_ids)
253
- residual = None
254
- for i in range(len(self.layers)):
255
- layer = self.layers[i]
256
- hidden_states, residual = layer(
257
- positions,
258
- hidden_states,
259
- kv_caches[i],
260
- input_metadata,
261
- residual,
262
- )
263
- hidden_states, _ = self.norm(hidden_states, residual)
264
- return hidden_states
265
-
266
-
267
-class MistralForCausalLM(nn.Module):
268
- packed_modules_mapping = {
269
- "qkv_proj": [
270
- "q_proj",
271
- "k_proj",
272
- "v_proj",
273
- ],
274
- "gate_up_proj": [
275
- "gate_proj",
276
- "up_proj",
277
- ],
278
- }
279
-
280
- # LoRA specific attributes
281
- supported_lora_modules = [
282
- "qkv_proj",
283
- "o_proj",
284
- "gate_up_proj",
285
- "down_proj",
286
- "embed_tokens",
287
- "lm_head",
288
- ]
289
- embedding_modules = {
290
- "embed_tokens": "input_embeddings",
291
- "lm_head": "output_embeddings",
292
- }
293
- embedding_padding_modules = ["lm_head"]
294
-
295
- def __init__(
296
- self,
297
- config: MistralConfig,
298
- linear_method: Optional[LinearMethodBase] = None,
299
- lora_config: Optional[LoRAConfig] = None,
300
- ) -> None:
301
- super().__init__()
302
- self.config = config
303
- self.linear_method = linear_method
304
- self.model = MistralModel(config,
305
- linear_method,
306
- lora_config=lora_config)
307
- unpadded_vocab_size = config.vocab_size
308
- if lora_config:
309
- unpadded_vocab_size += lora_config.lora_extra_vocab_size
310
- self.lm_head = ParallelLMHead(
311
- unpadded_vocab_size,
312
- config.hidden_size,
313
- org_num_embeddings=config.vocab_size,
314
- padding_size=DEFAULT_VOCAB_PADDING_SIZE
315
- # We need bigger padding if using lora for kernel
316
- # compatibility
317
- if not lora_config else lora_config.lora_vocab_padding_size,
318
- )
319
- self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
320
-
321
- def forward(
322
- self,
323
- input_ids: torch.Tensor,
324
- positions: torch.Tensor,
325
- kv_caches: List[KVCache],
326
- input_metadata: InputMetadata,
327
- ) -> torch.Tensor:
328
- hidden_states = self.model(input_ids, positions, kv_caches,
329
- input_metadata)
330
- return hidden_states
331
-
332
- def sample(
333
- self,
334
- hidden_states: torch.Tensor,
335
- sampling_metadata: SamplingMetadata,
336
- ) -> Optional[SamplerOutput]:
337
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
338
- sampling_metadata)
339
- return next_tokens
340
-
341
- def load_weights(self,
342
- model_name_or_path: str,
343
- cache_dir: Optional[str] = None,
344
- load_format: str = "auto",
345
- revision: Optional[str] = None):
346
- stacked_params_mapping = [
347
- # (param_name, shard_name, shard_id)
348
- ("qkv_proj", "q_proj", "q"),
349
- ("qkv_proj", "k_proj", "k"),
350
- ("qkv_proj", "v_proj", "v"),
351
- ("gate_up_proj", "gate_proj", 0),
352
- ("gate_up_proj", "up_proj", 1),
353
- ]
354
- params_dict = dict(self.named_parameters())
355
- for name, loaded_weight in hf_model_weights_iterator(
356
- model_name_or_path, cache_dir, load_format, revision):
357
- if "rotary_emb.inv_freq" in name:
358
- continue
359
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
360
- if weight_name not in name:
361
- continue
362
- name = name.replace(weight_name, param_name)
363
- # Skip loading extra bias for GPTQ models.
364
- if name.endswith(".bias") and name not in params_dict:
365
- continue
366
- param = params_dict[name]
367
- weight_loader = param.weight_loader
368
- weight_loader(param, loaded_weight, shard_id)
369
- break
370
- else:
371
- # Skip loading extra bias for GPTQ models.
372
- if name.endswith(".bias") and name not in params_dict:
373
- continue
374
- param = params_dict[name]
375
- weight_loader = getattr(param, "weight_loader",
376
- default_weight_loader)
377
- weight_loader(param, loaded_weight)