A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[misc] remove xverse modeling file (#10814)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 1, 2024 at 19:27 UTC
63a164172dbcc43857dbcf6443a7594faa143151
2 files changed
+1
-424
vllm/model_executor/models/registry.py
+1
-1
@@ -94,7 +94,7 @@ _TEXT_GENERATION_MODELS = {
94
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
95
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
96
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
97
- "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
97
+ "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
98
# [Encoder-decoder]
99
"BartModel": ("bart", "BartForConditionalGeneration"),
100
"BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/xverse.py
deleted
-423
@@ -1,423 +0,0 @@
1
-# Adapted from
2
-# https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py
3
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
4
-#
5
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
6
-# and OPT implementations in this library. It has been modified from its
7
-# original forms to accommodate minor architectural differences compared
8
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
9
-#
10
-# Licensed under the Apache License, Version 2.0 (the "License");
11
-# you may not use this file except in compliance with the License.
12
-# You may obtain a copy of the License at
13
-#
14
-# http://www.apache.org/licenses/LICENSE-2.0
15
-#
16
-# Unless required by applicable law or agreed to in writing, software
17
-# distributed under the License is distributed on an "AS IS" BASIS,
18
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
-# See the License for the specific language governing permissions and
20
-# limitations under the License.
21
-"""Inference-only Xverse model compatible with HuggingFace weights."""
22
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
23
-
24
-import torch
25
-from torch import nn
26
-from transformers import PretrainedConfig
27
-
28
-from vllm.attention import Attention, AttentionMetadata
29
-from vllm.compilation.decorators import support_torch_compile
30
-from vllm.config import CacheConfig, VllmConfig
31
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
-from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35
- QKVParallelLinear,
36
- RowParallelLinear)
37
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
-from vllm.model_executor.layers.quantization import QuantizationConfig
39
-from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41
-from vllm.model_executor.layers.vocab_parallel_embedding import (
42
- ParallelLMHead, VocabParallelEmbedding)
43
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
-from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.sequence import IntermediateTensors
46
-
47
-from .interfaces import SupportsLoRA, SupportsPP
48
-from .utils import (is_pp_missing_parameter,
49
- make_empty_intermediate_tensors_factory, make_layers,
50
- maybe_prefix)
51
-
52
-
53
-class XverseMLP(nn.Module):
54
-
55
- def __init__(
56
- self,
57
- hidden_size: int,
58
- intermediate_size: int,
59
- hidden_act: str,
60
- quant_config: Optional[QuantizationConfig] = None,
61
- ) -> None:
62
- super().__init__()
63
- self.gate_up_proj = MergedColumnParallelLinear(
64
- hidden_size, [intermediate_size] * 2,
65
- bias=False,
66
- quant_config=quant_config)
67
- self.down_proj = RowParallelLinear(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- quant_config=quant_config)
71
- if hidden_act != "silu":
72
- raise ValueError(f"Unsupported activation: {hidden_act}. "
73
- "Only silu is supported for now.")
74
- self.act_fn = SiluAndMul()
75
-
76
- def forward(self, x):
77
- gate, _ = self.gate_up_proj(x)
78
- x = self.act_fn(gate)
79
- x, _ = self.down_proj(x)
80
- return x
81
-
82
-
83
-class XverseAttention(nn.Module):
84
-
85
- def __init__(
86
- self,
87
- hidden_size: int,
88
- num_heads: int,
89
- num_kv_heads: int,
90
- rope_theta: float = 10000,
91
- rope_scaling: Optional[Dict[str, Any]] = None,
92
- max_position_embeddings: int = 8192,
93
- quant_config: Optional[QuantizationConfig] = None,
94
- bias: bool = False,
95
- cache_config: Optional[CacheConfig] = None,
96
- prefix: str = "",
97
- ) -> None:
98
- super().__init__()
99
- self.hidden_size = hidden_size
100
- tp_size = get_tensor_model_parallel_world_size()
101
- self.total_num_heads = num_heads
102
- assert self.total_num_heads % tp_size == 0
103
- self.num_heads = self.total_num_heads // tp_size
104
- self.total_num_kv_heads = num_kv_heads
105
- # partition the KV heads across multiple tensor parallel GPUs.
106
- assert self.total_num_kv_heads % tp_size == 0
107
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
108
- self.head_dim = hidden_size // self.total_num_heads
109
- self.q_size = self.num_heads * self.head_dim
110
- self.kv_size = self.num_kv_heads * self.head_dim
111
- self.scaling = self.head_dim**-0.5
112
- self.rope_theta = rope_theta
113
- self.max_position_embeddings = max_position_embeddings
114
-
115
- self.qkv_proj = QKVParallelLinear(
116
- hidden_size,
117
- self.head_dim,
118
- self.total_num_heads,
119
- self.total_num_kv_heads,
120
- bias=bias,
121
- quant_config=quant_config,
122
- )
123
- self.o_proj = RowParallelLinear(
124
- self.total_num_heads * self.head_dim,
125
- hidden_size,
126
- bias=bias,
127
- quant_config=quant_config,
128
- )
129
-
130
- self.rotary_emb = get_rope(
131
- self.head_dim,
132
- rotary_dim=self.head_dim,
133
- max_position=max_position_embeddings,
134
- base=rope_theta,
135
- rope_scaling=rope_scaling,
136
- )
137
- self.attn = Attention(self.num_heads,
138
- self.head_dim,
139
- self.scaling,
140
- num_kv_heads=self.num_kv_heads,
141
- cache_config=cache_config,
142
- quant_config=quant_config,
143
- prefix=f"{prefix}.attn")
144
-
145
- def forward(
146
- self,
147
- positions: torch.Tensor,
148
- hidden_states: torch.Tensor,
149
- kv_cache: torch.Tensor,
150
- attn_metadata: AttentionMetadata,
151
- ) -> torch.Tensor:
152
- qkv, _ = self.qkv_proj(hidden_states)
153
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
154
- q, k = self.rotary_emb(positions, q, k)
155
- attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
156
- output, _ = self.o_proj(attn_output)
157
- return output
158
-
159
-
160
-class XverseDecoderLayer(nn.Module):
161
-
162
- def __init__(
163
- self,
164
- config: PretrainedConfig,
165
- cache_config: Optional[CacheConfig] = None,
166
- quant_config: Optional[QuantizationConfig] = None,
167
- prefix: str = "",
168
- ) -> None:
169
- super().__init__()
170
- self.hidden_size = config.hidden_size
171
- rope_theta = getattr(config, "rope_theta", 10000)
172
- rope_scaling = getattr(config, "rope_scaling", None)
173
- max_position_embeddings = getattr(config, "max_position_embeddings",
174
- 8192)
175
- self.self_attn = XverseAttention(
176
- hidden_size=self.hidden_size,
177
- num_heads=config.num_attention_heads,
178
- num_kv_heads=getattr(config, "num_key_value_heads",
179
- config.num_attention_heads),
180
- rope_theta=rope_theta,
181
- rope_scaling=rope_scaling,
182
- max_position_embeddings=max_position_embeddings,
183
- quant_config=quant_config,
184
- bias=getattr(config, "bias", False),
185
- cache_config=cache_config,
186
- prefix=f"{prefix}.self_attn",
187
- )
188
- self.mlp = XverseMLP(
189
- hidden_size=self.hidden_size,
190
- intermediate_size=config.intermediate_size,
191
- hidden_act=config.hidden_act,
192
- quant_config=quant_config,
193
- )
194
- self.input_layernorm = RMSNorm(config.hidden_size,
195
- eps=config.rms_norm_eps)
196
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
197
- eps=config.rms_norm_eps)
198
-
199
- def forward(
200
- self,
201
- positions: torch.Tensor,
202
- hidden_states: torch.Tensor,
203
- kv_cache: torch.Tensor,
204
- attn_metadata: AttentionMetadata,
205
- residual: Optional[torch.Tensor],
206
- ) -> Tuple[torch.Tensor, torch.Tensor]:
207
- # Self Attention
208
- if residual is None:
209
- residual = hidden_states
210
- hidden_states = self.input_layernorm(hidden_states)
211
- else:
212
- hidden_states, residual = self.input_layernorm(
213
- hidden_states, residual)
214
- hidden_states = self.self_attn(
215
- positions=positions,
216
- hidden_states=hidden_states,
217
- kv_cache=kv_cache,
218
- attn_metadata=attn_metadata,
219
- )
220
-
221
- # Fully Connected
222
- hidden_states, residual = self.post_attention_layernorm(
223
- hidden_states, residual)
224
- hidden_states = self.mlp(hidden_states)
225
- return hidden_states, residual
226
-
227
-
228
-@support_torch_compile
229
-class XverseModel(nn.Module):
230
-
231
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
232
- super().__init__()
233
- config = vllm_config.model_config.hf_config
234
- cache_config = vllm_config.cache_config
235
- quant_config = vllm_config.quant_config
236
- lora_config = vllm_config.lora_config
237
- self.config = config
238
- self.padding_idx = config.pad_token_id
239
- lora_vocab = (lora_config.lora_extra_vocab_size *
240
- (lora_config.max_loras or 1)) if lora_config else 0
241
- self.vocab_size = config.vocab_size + lora_vocab
242
- self.org_vocab_size = config.vocab_size
243
- self.embed_tokens = VocabParallelEmbedding(
244
- self.vocab_size,
245
- config.hidden_size,
246
- org_num_embeddings=config.vocab_size,
247
- )
248
- self.start_layer, self.end_layer, self.layers = make_layers(
249
- config.num_hidden_layers,
250
- lambda prefix: XverseDecoderLayer(
251
- config, cache_config, quant_config, prefix=prefix),
252
- prefix=f"{prefix}.layers",
253
- )
254
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
255
- self.make_empty_intermediate_tensors = (
256
- make_empty_intermediate_tensors_factory(
257
- ["hidden_states", "residual"], config.hidden_size))
258
-
259
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
260
- return self.embed_tokens(input_ids)
261
-
262
- def forward(
263
- self,
264
- input_ids: torch.Tensor,
265
- positions: torch.Tensor,
266
- kv_caches: List[torch.Tensor],
267
- attn_metadata: AttentionMetadata,
268
- intermediate_tensors: Optional[IntermediateTensors],
269
- inputs_embeds: Optional[torch.Tensor] = None,
270
- ) -> Union[torch.Tensor, IntermediateTensors]:
271
- if get_pp_group().is_first_rank:
272
- if inputs_embeds is not None:
273
- hidden_states = inputs_embeds
274
- else:
275
- hidden_states = self.get_input_embeddings(input_ids)
276
- residual = None
277
- else:
278
- hidden_states = intermediate_tensors["hidden_states"]
279
- residual = intermediate_tensors["residual"]
280
- for i in range(self.start_layer, self.end_layer):
281
- layer = self.layers[i]
282
- hidden_states, residual = layer(
283
- positions,
284
- hidden_states,
285
- kv_caches[i - self.start_layer],
286
- attn_metadata,
287
- residual,
288
- )
289
- if not get_pp_group().is_last_rank:
290
- return IntermediateTensors({
291
- "hidden_states": hidden_states,
292
- "residual": residual
293
- })
294
- hidden_states, _ = self.norm(hidden_states, residual)
295
- return hidden_states
296
-
297
-
298
-class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
299
- packed_modules_mapping = {
300
- "qkv_proj": [
301
- "q_proj",
302
- "k_proj",
303
- "v_proj",
304
- ],
305
- "gate_up_proj": [
306
- "gate_proj",
307
- "up_proj",
308
- ],
309
- }
310
-
311
- # LoRA specific attributes
312
- supported_lora_modules = [
313
- "qkv_proj",
314
- "o_proj",
315
- "gate_up_proj",
316
- "down_proj",
317
- "embed_tokens",
318
- "lm_head",
319
- ]
320
- embedding_modules = {
321
- "embed_tokens": "input_embeddings",
322
- "lm_head": "output_embeddings",
323
- }
324
- embedding_padding_modules = ["lm_head"]
325
-
326
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
327
- super().__init__()
328
-
329
- config = vllm_config.model_config.hf_config
330
- quant_config = vllm_config.quant_config
331
- lora_config = vllm_config.lora_config
332
-
333
- self.config = config
334
- self.lora_config = lora_config
335
-
336
- self.quant_config = quant_config
337
- self.model = XverseModel(vllm_config=vllm_config,
338
- prefix=maybe_prefix(prefix, "model"))
339
- self.lm_head = ParallelLMHead(config.vocab_size,
340
- config.hidden_size,
341
- quant_config=quant_config)
342
- if self.config.tie_word_embeddings:
343
- self.lm_head.weight = self.model.embed_tokens.weight
344
- self.logits_processor = LogitsProcessor(config.vocab_size)
345
- self.sampler = get_sampler()
346
- self.make_empty_intermediate_tensors = (
347
- self.model.make_empty_intermediate_tensors)
348
-
349
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
350
- return self.model.get_input_embeddings(input_ids)
351
-
352
- def forward(
353
- self,
354
- input_ids: torch.Tensor,
355
- positions: torch.Tensor,
356
- kv_caches: List[torch.Tensor],
357
- attn_metadata: AttentionMetadata,
358
- intermediate_tensors: Optional[IntermediateTensors] = None,
359
- inputs_embeds: Optional[torch.Tensor] = None,
360
- ) -> Union[torch.Tensor, IntermediateTensors]:
361
- hidden_states = self.model(input_ids, positions, kv_caches,
362
- attn_metadata, intermediate_tensors,
363
- inputs_embeds)
364
- return hidden_states
365
-
366
- def compute_logits(
367
- self,
368
- hidden_states: torch.Tensor,
369
- sampling_metadata: SamplingMetadata,
370
- ) -> Optional[torch.Tensor]:
371
- logits = self.logits_processor(self.lm_head, hidden_states,
372
- sampling_metadata)
373
- return logits
374
-
375
- def sample(
376
- self,
377
- logits: torch.Tensor,
378
- sampling_metadata: SamplingMetadata,
379
- ) -> Optional[SamplerOutput]:
380
- next_tokens = self.sampler(logits, sampling_metadata)
381
- return next_tokens
382
-
383
- def load_weights(self, weights: Iterable[Tuple[str,
384
- torch.Tensor]]) -> Set[str]:
385
- stacked_params_mapping = [
386
- ("qkv_proj", "q_proj", "q"),
387
- ("qkv_proj", "k_proj", "k"),
388
- ("qkv_proj", "v_proj", "v"),
389
- ("gate_up_proj", "gate_proj", 0),
390
- ("gate_up_proj", "up_proj", 1),
391
- ]
392
- params_dict = dict(self.named_parameters())
393
- loaded_params: Set[str] = set()
394
- for name, loaded_weight in weights:
395
- if ("rotary_emb.inv_freq" in name
396
- or "rotary_emb.cos_cached" in name
397
- or "rotary_emb.sin_cached" in name):
398
- continue
399
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
400
- if weight_name not in name:
401
- continue
402
- name = name.replace(weight_name, param_name)
403
- # Skip loading extra bias for GPTQ models.
404
- if name.endswith(".bias") and name not in params_dict:
405
- continue
406
- if is_pp_missing_parameter(name, self):
407
- continue
408
- param = params_dict[name]
409
- weight_loader = param.weight_loader
410
- weight_loader(param, loaded_weight, shard_id)
411
- break
412
- else:
413
- # Skip loading extra bias for GPTQ models.
414
- if name.endswith(".bias") and name not in params_dict:
415
- continue
416
- if is_pp_missing_parameter(name, self):
417
- continue
418
- param = params_dict[name]
419
- weight_loader = getattr(param, "weight_loader",
420
- default_weight_loader)
421
- weight_loader(param, loaded_weight)
422
- loaded_params.add(name)
423
- return loaded_params