A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Support Nemotron models (Nemotron-3, Nemotron-4, Minitron) (#6611)
Michael Goin committed
Jul 26, 2024 at 14:33 UTC
07278c37ddd898d842bbddc382e4f67ac08dae35
9 files changed
+776
-1
.buildkite/lm-eval-harness/configs/Minitron-4B-Base.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nvidia/Minitron-4B-Base -b auto -l 1000 -f 5 -t 1
2
+model_name: "nvidia/Minitron-4B-Base"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.252
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.252
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -4,5 +4,6 @@ Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
+Minitron-4B-Base.yaml
8
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9
Qwen2-1.5B-Instruct-FP8W8.yaml
vllm/model_executor/layers/activation.py
+16
@@ -159,6 +159,21 @@ class QuickGELU(CustomOp):
159
# def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
160
161
162
+class ReLUSquaredActivation(CustomOp):
163
+ """
164
+ Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
165
+ """
166
+
167
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
168
+ """PyTorch-native implementation equivalent to forward()."""
169
+ relu_applied = nn.functional.relu(x)
170
+ squared = torch.square(relu_applied)
171
+ return squared
172
+
173
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
174
+ return self.forward_native(x)
175
+
176
+
177
class ScaledActivation(nn.Module):
178
"""An activation function with post-scale parameters.
179
@@ -207,6 +222,7 @@ _ACTIVATION_REGISTRY = {
222
"gelu_new": NewGELU(),
223
"gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
224
"relu": nn.ReLU(),
225
+ "relu2": ReLUSquaredActivation(),
226
"quick_gelu": QuickGELU(),
227
}
228
vllm/model_executor/layers/rotary_embedding.py
+3
@@ -774,6 +774,7 @@ def get_rope(
774
is_neox_style: bool = True,
775
rope_scaling: Optional[Dict[str, Any]] = None,
776
dtype: Optional[torch.dtype] = None,
777
+ rotary_percent: float = 1.0,
778
) -> RotaryEmbedding:
779
if dtype is None:
780
dtype = torch.get_default_dtype()
@@ -786,6 +787,8 @@ def get_rope(
787
rope_scaling_args = tuple(rope_scaling_tuple.items())
788
else:
789
rope_scaling_args = None
790
+ if rotary_percent < 1.0:
791
+ rotary_dim = int(rotary_dim * rotary_percent)
792
key = (head_size, rotary_dim, max_position, base, is_neox_style,
793
rope_scaling_args, dtype)
794
if key in _ROPE_DICT:
vllm/model_executor/models/__init__.py
+1
@@ -51,6 +51,7 @@ _GENERATION_MODELS = {
51
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
52
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
53
"MiniCPMV": ("minicpmv", "MiniCPMV"),
54
+ "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
55
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
56
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
57
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/nemotron.py
new
+531
@@ -0,0 +1,531 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Nemotron model compatible with HuggingFace weights."""
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25
+
26
+import torch
27
+from torch import nn
28
+
29
+from vllm.attention import Attention, AttentionMetadata
30
+from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32
+from vllm.model_executor.layers.activation import get_act_fn
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ QKVParallelLinear,
35
+ RowParallelLinear)
36
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
+from vllm.model_executor.layers.quantization.base_config import (
38
+ QuantizationConfig)
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.vocab_parallel_embedding import (
42
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
+from vllm.model_executor.model_loader.weight_utils import (
44
+ default_weight_loader, maybe_remap_kv_scale_name)
45
+from vllm.model_executor.sampling_metadata import SamplingMetadata
46
+from vllm.sequence import IntermediateTensors, SamplerOutput
47
+from vllm.transformers_utils.configs import NemotronConfig
48
+
49
+from .interfaces import SupportsLoRA
50
+from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
51
+
52
+# The architecture is pretty similar to Llama, with these changes:
53
+# - There is no gate_proj, just up_proj
54
+# - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm
55
+# - Squared ReLU instead of SwiGLU
56
+# - Adds a rotary_percent to RoPE
57
+
58
+
59
+def _cast_if_autocast_enabled(*args):
60
+ if not torch.is_autocast_enabled():
61
+ return args
62
+ else:
63
+ return torch.cuda.amp.autocast_mode._cast(
64
+ args, torch.get_autocast_gpu_dtype())
65
+
66
+
67
+class NemotronLayerNorm1P(nn.LayerNorm):
68
+
69
+ def __init__(self,
70
+ normalized_shape: Union[int, List[int], torch.Size],
71
+ eps: float = 1e-5,
72
+ elementwise_affine: bool = True,
73
+ bias: bool = True,
74
+ device=None,
75
+ dtype=None):
76
+ super().__init__(normalized_shape, eps, elementwise_affine, bias,
77
+ device, dtype)
78
+
79
+ def forward(
80
+ self,
81
+ x: torch.Tensor,
82
+ residual: Optional[torch.Tensor] = None,
83
+ ) -> torch.Tensor:
84
+ if residual is not None:
85
+ x = x + residual
86
+ residual = x
87
+ args = _cast_if_autocast_enabled(x, self.normalized_shape,
88
+ self.weight + 1, self.bias, self.eps)
89
+ with torch.cuda.amp.autocast(enabled=False):
90
+ x = torch.nn.functional.layer_norm(*args)
91
+ return x if residual is None else (x, residual)
92
+
93
+
94
+class NemotronMLP(nn.Module):
95
+
96
+ def __init__(
97
+ self,
98
+ hidden_size: int,
99
+ intermediate_size: int,
100
+ hidden_act: str,
101
+ quant_config: Optional[QuantizationConfig] = None,
102
+ bias: bool = False,
103
+ prefix: str = "",
104
+ ) -> None:
105
+ super().__init__()
106
+ self.up_proj = ColumnParallelLinear(input_size=hidden_size,
107
+ output_size=intermediate_size,
108
+ bias=bias,
109
+ quant_config=quant_config,
110
+ prefix=f"{prefix}.up_proj")
111
+ self.down_proj = RowParallelLinear(input_size=intermediate_size,
112
+ output_size=hidden_size,
113
+ bias=bias,
114
+ quant_config=quant_config,
115
+ prefix=f"{prefix}.down_proj")
116
+ self.act_fn = get_act_fn(hidden_act)
117
+
118
+ def forward(self, x):
119
+ up, _ = self.up_proj(x)
120
+ x = self.act_fn(up)
121
+ x, _ = self.down_proj(x)
122
+ return x
123
+
124
+
125
+class NemotronAttention(nn.Module):
126
+
127
+ def __init__(
128
+ self,
129
+ config: NemotronConfig,
130
+ hidden_size: int,
131
+ num_heads: int,
132
+ num_kv_heads: int,
133
+ rope_theta: float = 10000,
134
+ rope_scaling: Optional[Dict[str, Any]] = None,
135
+ max_position_embeddings: int = 8192,
136
+ quant_config: Optional[QuantizationConfig] = None,
137
+ bias: bool = False,
138
+ cache_config: Optional[CacheConfig] = None,
139
+ prefix: str = "",
140
+ ) -> None:
141
+ super().__init__()
142
+ self.hidden_size = hidden_size
143
+ tp_size = get_tensor_model_parallel_world_size()
144
+ self.total_num_heads = num_heads
145
+ assert self.total_num_heads % tp_size == 0
146
+ self.num_heads = self.total_num_heads // tp_size
147
+ self.total_num_kv_heads = num_kv_heads
148
+ if self.total_num_kv_heads >= tp_size:
149
+ # Number of KV heads is greater than TP size, so we partition
150
+ # the KV heads across multiple tensor parallel GPUs.
151
+ assert self.total_num_kv_heads % tp_size == 0
152
+ else:
153
+ # Number of KV heads is less than TP size, so we replicate
154
+ # the KV heads across multiple tensor parallel GPUs.
155
+ assert tp_size % self.total_num_kv_heads == 0
156
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
157
+ # MistralConfig has an optional head_dim introduced by Mistral-Nemo
158
+ self.head_dim = getattr(config, "head_dim",
159
+ self.hidden_size // self.total_num_heads)
160
+ self.q_size = self.num_heads * self.head_dim
161
+ self.kv_size = self.num_kv_heads * self.head_dim
162
+ self.scaling = self.head_dim**-0.5
163
+ self.rope_theta = rope_theta
164
+ self.rotary_percent = config.rope_percent
165
+ self.max_position_embeddings = max_position_embeddings
166
+
167
+ self.qkv_proj = QKVParallelLinear(
168
+ hidden_size=hidden_size,
169
+ head_size=self.head_dim,
170
+ total_num_heads=self.total_num_heads,
171
+ total_num_kv_heads=self.total_num_kv_heads,
172
+ bias=bias,
173
+ quant_config=quant_config,
174
+ prefix=f"{prefix}.qkv_proj",
175
+ )
176
+ self.o_proj = RowParallelLinear(
177
+ input_size=self.total_num_heads * self.head_dim,
178
+ output_size=hidden_size,
179
+ bias=bias,
180
+ quant_config=quant_config,
181
+ prefix=f"{prefix}.o_proj",
182
+ )
183
+
184
+ self.rotary_emb = get_rope(
185
+ self.head_dim,
186
+ rotary_dim=self.head_dim,
187
+ max_position=max_position_embeddings,
188
+ base=rope_theta,
189
+ rope_scaling=rope_scaling,
190
+ rotary_percent=self.rotary_percent,
191
+ )
192
+ self.attn = Attention(self.num_heads,
193
+ self.head_dim,
194
+ self.scaling,
195
+ num_kv_heads=self.num_kv_heads,
196
+ cache_config=cache_config,
197
+ quant_config=quant_config)
198
+
199
+ def forward(
200
+ self,
201
+ positions: torch.Tensor,
202
+ hidden_states: torch.Tensor,
203
+ kv_cache: torch.Tensor,
204
+ attn_metadata: AttentionMetadata,
205
+ ) -> torch.Tensor:
206
+ qkv, _ = self.qkv_proj(hidden_states)
207
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
208
+ q, k = self.rotary_emb(positions, q, k)
209
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
210
+ output, _ = self.o_proj(attn_output)
211
+ return output
212
+
213
+
214
+class NemotronDecoderLayer(nn.Module):
215
+
216
+ def __init__(
217
+ self,
218
+ config: NemotronConfig,
219
+ cache_config: Optional[CacheConfig] = None,
220
+ quant_config: Optional[QuantizationConfig] = None,
221
+ prefix: str = "",
222
+ ) -> None:
223
+ super().__init__()
224
+ self.hidden_size = config.hidden_size
225
+ rope_theta = getattr(config, "rope_theta", 10000)
226
+ rope_scaling = getattr(config, "rope_scaling", None)
227
+ if rope_scaling is not None and getattr(
228
+ config, "original_max_position_embeddings", None):
229
+ rope_scaling["original_max_position_embeddings"] = (
230
+ config.original_max_position_embeddings)
231
+ max_position_embeddings = getattr(config, "max_position_embeddings",
232
+ 8192)
233
+ # Support abacusai/Smaug-72B-v0.1 with attention_bias
234
+ # Support internlm/internlm-7b with bias
235
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
236
+ config, "bias", False)
237
+ self.self_attn = NemotronAttention(
238
+ config=config,
239
+ hidden_size=self.hidden_size,
240
+ num_heads=config.num_attention_heads,
241
+ num_kv_heads=getattr(config, "num_key_value_heads",
242
+ config.num_attention_heads),
243
+ rope_theta=rope_theta,
244
+ rope_scaling=rope_scaling,
245
+ max_position_embeddings=max_position_embeddings,
246
+ quant_config=quant_config,
247
+ bias=attention_bias,
248
+ cache_config=cache_config,
249
+ prefix=f"{prefix}.self_attn",
250
+ )
251
+ self.mlp = NemotronMLP(
252
+ hidden_size=self.hidden_size,
253
+ intermediate_size=config.intermediate_size,
254
+ hidden_act=config.hidden_act,
255
+ quant_config=quant_config,
256
+ bias=getattr(config, "mlp_bias", False),
257
+ prefix=f"{prefix}.mlp",
258
+ )
259
+ self.input_layernorm = NemotronLayerNorm1P(config.hidden_size,
260
+ eps=config.norm_eps)
261
+ self.post_attention_layernorm = NemotronLayerNorm1P(
262
+ config.hidden_size, eps=config.norm_eps)
263
+
264
+ def forward(
265
+ self,
266
+ positions: torch.Tensor,
267
+ hidden_states: torch.Tensor,
268
+ kv_cache: torch.Tensor,
269
+ attn_metadata: AttentionMetadata,
270
+ residual: Optional[torch.Tensor],
271
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
272
+ # Self Attention
273
+ if residual is None:
274
+ residual = hidden_states
275
+ hidden_states = self.input_layernorm(hidden_states)
276
+ else:
277
+ hidden_states, residual = self.input_layernorm(
278
+ hidden_states, residual)
279
+ hidden_states = self.self_attn(
280
+ positions=positions,
281
+ hidden_states=hidden_states,
282
+ kv_cache=kv_cache,
283
+ attn_metadata=attn_metadata,
284
+ )
285
+
286
+ # Fully Connected
287
+ hidden_states, residual = self.post_attention_layernorm(
288
+ hidden_states, residual)
289
+ hidden_states = self.mlp(hidden_states)
290
+ return hidden_states, residual
291
+
292
+
293
+class NemotronModel(nn.Module):
294
+
295
+ def __init__(
296
+ self,
297
+ config: NemotronConfig,
298
+ cache_config: Optional[CacheConfig] = None,
299
+ quant_config: Optional[QuantizationConfig] = None,
300
+ lora_config: Optional[LoRAConfig] = None,
301
+ prefix: str = "",
302
+ ) -> None:
303
+ super().__init__()
304
+ self.config = config
305
+ self.padding_idx = config.pad_token_id
306
+ lora_vocab = (lora_config.lora_extra_vocab_size *
307
+ (lora_config.max_loras or 1)) if lora_config else 0
308
+ self.vocab_size = config.vocab_size + lora_vocab
309
+ self.org_vocab_size = config.vocab_size
310
+ if get_pp_group().is_first_rank or (config.tie_word_embeddings
311
+ and get_pp_group().is_last_rank):
312
+ self.embed_tokens = VocabParallelEmbedding(
313
+ self.vocab_size,
314
+ config.hidden_size,
315
+ org_num_embeddings=config.vocab_size,
316
+ )
317
+ else:
318
+ self.embed_tokens = PPMissingLayer()
319
+ self.start_layer, self.end_layer, self.layers = make_layers(
320
+ config.num_hidden_layers,
321
+ lambda prefix: NemotronDecoderLayer(config=config,
322
+ cache_config=cache_config,
323
+ quant_config=quant_config,
324
+ prefix=prefix),
325
+ prefix=f"{prefix}.layers")
326
+ if get_pp_group().is_last_rank:
327
+ self.norm = NemotronLayerNorm1P(config.hidden_size,
328
+ eps=config.norm_eps)
329
+ else:
330
+ self.norm = PPMissingLayer()
331
+
332
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
333
+ return self.embed_tokens(input_ids)
334
+
335
+ def forward(
336
+ self,
337
+ input_ids: Optional[torch.Tensor],
338
+ positions: torch.Tensor,
339
+ kv_caches: List[torch.Tensor],
340
+ attn_metadata: AttentionMetadata,
341
+ intermediate_tensors: Optional[IntermediateTensors],
342
+ inputs_embeds: Optional[torch.Tensor] = None,
343
+ ) -> Union[torch.Tensor, IntermediateTensors]:
344
+ if get_pp_group().is_first_rank:
345
+ if inputs_embeds is not None:
346
+ hidden_states = inputs_embeds
347
+ else:
348
+ hidden_states = self.get_input_embeddings(input_ids)
349
+ residual = None
350
+ else:
351
+ assert intermediate_tensors is not None
352
+ hidden_states = intermediate_tensors["hidden_states"]
353
+ residual = intermediate_tensors["residual"]
354
+
355
+ for i in range(self.start_layer, self.end_layer):
356
+ layer = self.layers[i]
357
+ hidden_states, residual = layer(
358
+ positions,
359
+ hidden_states,
360
+ kv_caches[i - self.start_layer],
361
+ attn_metadata,
362
+ residual,
363
+ )
364
+
365
+ if not get_pp_group().is_last_rank:
366
+ return IntermediateTensors({
367
+ "hidden_states": hidden_states,
368
+ "residual": residual
369
+ })
370
+
371
+ hidden_states, _ = self.norm(hidden_states, residual)
372
+ return hidden_states
373
+
374
+
375
+class NemotronForCausalLM(nn.Module, SupportsLoRA):
376
+ packed_modules_mapping = {
377
+ "qkv_proj": [
378
+ "q_proj",
379
+ "k_proj",
380
+ "v_proj",
381
+ ],
382
+ }
383
+
384
+ # LoRA specific attributes
385
+ supported_lora_modules = [
386
+ "qkv_proj", "o_proj", "up_proj", "down_proj", "embed_tokens", "lm_head"
387
+ ]
388
+ embedding_modules = {
389
+ "embed_tokens": "input_embeddings",
390
+ "lm_head": "output_embeddings",
391
+ }
392
+ embedding_padding_modules = ["lm_head"]
393
+ bitsandbytes_stacked_params_mapping = {
394
+ # shard_name, weight_name, index
395
+ "q_proj": ("qkv_proj", 0),
396
+ "k_proj": ("qkv_proj", 1),
397
+ "v_proj": ("qkv_proj", 2),
398
+ }
399
+
400
+ def __init__(
401
+ self,
402
+ config: NemotronConfig,
403
+ cache_config: Optional[CacheConfig] = None,
404
+ quant_config: Optional[QuantizationConfig] = None,
405
+ lora_config: Optional[LoRAConfig] = None,
406
+ ) -> None:
407
+ super().__init__()
408
+
409
+ assert isinstance(config, NemotronConfig)
410
+
411
+ self.config = config
412
+ self.lora_config = lora_config
413
+
414
+ self.model = NemotronModel(config,
415
+ cache_config,
416
+ quant_config,
417
+ lora_config=lora_config,
418
+ prefix="model")
419
+ if get_pp_group().is_last_rank:
420
+ self.unpadded_vocab_size = config.vocab_size
421
+ if lora_config:
422
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
423
+ self.lm_head = ParallelLMHead(
424
+ self.unpadded_vocab_size,
425
+ config.hidden_size,
426
+ org_num_embeddings=config.vocab_size,
427
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
428
+ # We need bigger padding if using lora for kernel
429
+ # compatibility
430
+ if not lora_config else lora_config.lora_vocab_padding_size,
431
+ quant_config=quant_config,
432
+ )
433
+ if config.tie_word_embeddings:
434
+ self.lm_head.weight = self.model.embed_tokens.weight
435
+
436
+ logit_scale = getattr(config, "logit_scale", 1.0)
437
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
438
+ config.vocab_size,
439
+ logit_scale)
440
+ self.sampler = Sampler()
441
+ else:
442
+ self.lm_head = PPMissingLayer()
443
+
444
+ def forward(
445
+ self,
446
+ input_ids: torch.Tensor,
447
+ positions: torch.Tensor,
448
+ kv_caches: List[torch.Tensor],
449
+ attn_metadata: AttentionMetadata,
450
+ intermediate_tensors: Optional[IntermediateTensors] = None,
451
+ ) -> Union[torch.Tensor, IntermediateTensors]:
452
+ model_output = self.model(input_ids, positions, kv_caches,
453
+ attn_metadata, intermediate_tensors)
454
+ return model_output
455
+
456
+ def compute_logits(self, hidden_states: torch.Tensor,
457
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
458
+ logits = self.logits_processor(self.lm_head, hidden_states,
459
+ sampling_metadata)
460
+ return logits
461
+
462
+ def sample(
463
+ self,
464
+ logits: torch.Tensor,
465
+ sampling_metadata: SamplingMetadata,
466
+ ) -> Optional[SamplerOutput]:
467
+ next_tokens = self.sampler(logits, sampling_metadata)
468
+ return next_tokens
469
+
470
+ def make_empty_intermediate_tensors(
471
+ self, batch_size: int, dtype: torch.dtype,
472
+ device: torch.device) -> IntermediateTensors:
473
+ return IntermediateTensors({
474
+ "hidden_states":
475
+ torch.zeros((batch_size, self.config.hidden_size),
476
+ dtype=dtype,
477
+ device=device),
478
+ "residual":
479
+ torch.zeros((batch_size, self.config.hidden_size),
480
+ dtype=dtype,
481
+ device=device),
482
+ })
483
+
484
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
485
+ stacked_params_mapping = [
486
+ # (param_name, shard_name, shard_id)
487
+ (".qkv_proj", ".q_proj", "q"),
488
+ (".qkv_proj", ".k_proj", "k"),
489
+ (".qkv_proj", ".v_proj", "v"),
490
+ ]
491
+ params_dict = dict(self.named_parameters())
492
+ for name, loaded_weight in weights:
493
+ if "rotary_emb.inv_freq" in name:
494
+ continue
495
+ if ("rotary_emb.cos_cached" in name
496
+ or "rotary_emb.sin_cached" in name):
497
+ # Models trained using ColossalAI may include these tensors in
498
+ # the checkpoint. Skip them.
499
+ continue
500
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
501
+ if weight_name not in name:
502
+ continue
503
+ name = name.replace(weight_name, param_name)
504
+ # Skip loading extra bias for GPTQ models.
505
+ if name.endswith(".bias") and name not in params_dict:
506
+ continue
507
+
508
+ if is_pp_missing_parameter(name, self):
509
+ continue
510
+
511
+ param = params_dict[name]
512
+ weight_loader = param.weight_loader
513
+ weight_loader(param, loaded_weight, shard_id)
514
+
515
+ break
516
+ else:
517
+ # Skip loading extra bias for GPTQ models.
518
+ if name.endswith(".bias") and name not in params_dict:
519
+ continue
520
+ # Remapping the name of FP8 kv-scale.
521
+ name = maybe_remap_kv_scale_name(name, params_dict)
522
+ if name is None:
523
+ continue
524
+
525
+ if is_pp_missing_parameter(name, self):
526
+ continue
527
+
528
+ param = params_dict[name]
529
+ weight_loader = getattr(param, "weight_loader",
530
+ default_weight_loader)
531
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+2
-1
@@ -8,7 +8,7 @@ from vllm.logger import init_logger
8
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
JAISConfig, MedusaConfig,
10
MLPSpeculatorConfig, MPTConfig,
11
- RWConfig)
11
+ NemotronConfig, RWConfig)
12
13
if VLLM_USE_MODELSCOPE:
14
from modelscope import AutoConfig
@@ -26,6 +26,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
26
"jais": JAISConfig,
27
"mlp_speculator": MLPSpeculatorConfig,
28
"medusa": MedusaConfig,
29
+ "nemotron": NemotronConfig,
30
}
31
32
for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -8,6 +8,7 @@ from vllm.transformers_utils.configs.jais import JAISConfig
8
from vllm.transformers_utils.configs.medusa import MedusaConfig
9
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
10
from vllm.transformers_utils.configs.mpt import MPTConfig
11
+from vllm.transformers_utils.configs.nemotron import NemotronConfig
12
13
__all__ = [
14
"ChatGLMConfig",
@@ -17,4 +18,5 @@ __all__ = [
18
"JAISConfig",
19
"MedusaConfig",
20
"MLPSpeculatorConfig",
21
+ "NemotronConfig",
22
]
vllm/transformers_utils/configs/nemotron.py
new
+209
@@ -0,0 +1,209 @@
1
+# coding=utf-8
2
+# Copyright 2024 HuggingFace Inc. team. All rights reserved.
3
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
4
+#
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+#
9
+# http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+"""Nemotron model configuration"""
17
+
18
+from transformers import PretrainedConfig
19
+from transformers.utils import logging
20
+
21
+logger = logging.get_logger(__name__)
22
+
23
+
24
+class NemotronConfig(PretrainedConfig):
25
+ r"""
26
+ This is the configuration class to store the configuration of a
27
+ [`NemotronModel`]. It is used to instantiate an Nemotron model
28
+ according to the specified arguments, defining the model architecture.
29
+ Instantiating a configuration with the defaults will yield a similar
30
+ configuration to that of the Nemotron-8B.
31
+
32
+ Configuration objects inherit from [`PretrainedConfig`] and can be
33
+ used to control the model outputs. Read the documentation from
34
+ [`PretrainedConfig`] for more information.
35
+
36
+
37
+ Args:
38
+ vocab_size (`int`, *optional*, defaults to 32000):
39
+ Vocabulary size of the Nemotron model. Defines the number of
40
+ different tokens that can be represented by the
41
+ `inputs_ids` passed when calling [`NemotronModel`]
42
+ hidden_size (`int`, *optional*, defaults to 4096):
43
+ Dimension of the hidden representations.
44
+ intermediate_size (`int`, *optional*, defaults to 11008):
45
+ Dimension of the MLP representations.
46
+ num_hidden_layers (`int`, *optional*, defaults to 32):
47
+ Number of hidden layers in the Transformer decoder.
48
+ num_attention_heads (`int`, *optional*, defaults to 32):
49
+ Number of attention heads for each attention layer in the
50
+ Transformer decoder.
51
+ head_dim (`int`, *optional*, defaults to None):
52
+ Projection weights dimension in multi-head attention. Set to
53
+ hidden_size // num_attention_heads if None
54
+ num_key_value_heads (`int`, *optional*):
55
+ This is the number of key_value heads that should be used to
56
+ implement Grouped Query Attention. If
57
+ `num_key_value_heads=num_attention_heads`, the model will use
58
+ Multi Head Attention (MHA), if
59
+ `num_key_value_heads=1 the model will use Multi Query Attention
60
+ (MQA) otherwise GQA is used. When converting a multi-head
61
+ checkpoint to a GQA checkpoint, each group key and value
62
+ head should be constructed by meanpooling all the original
63
+ heads within that group. For more details checkout
64
+ [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
65
+ is not specified, will default to `num_attention_heads`.
66
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
67
+ The non-linear activation function (function or string) in the
68
+ decoder.
69
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
70
+ The maximum sequence length that this model might ever be used
71
+ with.
72
+ initializer_range (`float`, *optional*, defaults to 0.02):
73
+ The standard deviation of the truncated_normal_initializer for
74
+ initializing all weight matrices.
75
+ norm_eps (`float`, *optional*, defaults to 1e-06):
76
+ The epsilon used by the normalization layers.
77
+ use_cache (`bool`, *optional*, defaults to `True`):
78
+ Whether or not the model should return the last key/values
79
+ attentions (not used by all models). Only relevant if
80
+ `config.is_decoder=True`.
81
+ pad_token_id (`int`, *optional*):
82
+ Padding token id.
83
+ bos_token_id (`int`, *optional*, defaults to 1):
84
+ Beginning of stream token id.
85
+ eos_token_id (`int`, *optional*, defaults to 2):
86
+ End of stream token id.
87
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88
+ Whether to tie weight embeddings
89
+ rope_theta (`float`, *optional*, defaults to 10000.0):
90
+ The base period of the RoPE embeddings.
91
+ rope_scaling (`Dict`, *optional*):
92
+ Dictionary containing the scaling configuration for the RoPE
93
+ embeddings. Currently supports two scaling strategies: linear
94
+ and dynamic. Their scaling factor must be a float greater than 1.
95
+ The expected format is `{"type": strategy name,
96
+ "factor": scaling factor}`. When using this flag, don't update
97
+ `max_position_embeddings` to the expected new maximum.
98
+ attention_bias (`bool`, *optional*, defaults to `False`):
99
+ Whether to use a bias in the query, key, value and output
100
+ projection layers during self-attention.
101
+ attention_dropout (`float`, *optional*, defaults to 0.0):
102
+ The dropout ratio for the attention probabilities.
103
+ mlp_bias (`bool`, *optional*, defaults to `False`):
104
+ Whether to use a bias in up_proj and down_proj layers in the MLP
105
+ layers.
106
+
107
+ ```python
108
+ >>> from transformers import NemotronModel, NemotronConfig
109
+
110
+ >>> # Initializing a Nemotron nemotron-15b style configuration
111
+ >>> configuration = NemotronConfig()
112
+
113
+ >>> # Initializing a model from the nemotron-15b style configuration
114
+ >>> model = NemotronModel(configuration)
115
+
116
+ >>> # Accessing the model configuration
117
+ >>> configuration = model.config
118
+ ```"""
119
+
120
+ model_type = "nemotron"
121
+ keys_to_ignore_at_inference = ["past_key_values"]
122
+
123
+ def __init__(
124
+ self,
125
+ vocab_size=256000,
126
+ hidden_size=6144,
127
+ intermediate_size=24576,
128
+ num_hidden_layers=32,
129
+ num_attention_heads=48,
130
+ head_dim=None,
131
+ num_key_value_heads=None,
132
+ hidden_act="relu2",
133
+ max_position_embeddings=4096,
134
+ initializer_range=0.0134,
135
+ norm_eps=1e-5,
136
+ use_cache=True,
137
+ pad_token_id=None,
138
+ bos_token_id=2,
139
+ eos_token_id=3,
140
+ tie_word_embeddings=False,
141
+ rope_theta=10000.0,
142
+ rope_scaling=None,
143
+ rope_percent=0.5,
144
+ attention_bias=False,
145
+ attention_dropout=0.0,
146
+ mlp_bias=False,
147
+ **kwargs,
148
+ ):
149
+ self.vocab_size = vocab_size
150
+ self.max_position_embeddings = max_position_embeddings
151
+ self.hidden_size = hidden_size
152
+ self.intermediate_size = intermediate_size
153
+ self.num_hidden_layers = num_hidden_layers
154
+ self.num_attention_heads = num_attention_heads
155
+ head_dim = head_dim or kwargs.get("kv_channels", None)
156
+ self.head_dim = head_dim if head_dim is not None else (
157
+ hidden_size // num_attention_heads)
158
+
159
+ # for backward compatibility
160
+ if num_key_value_heads is None:
161
+ num_key_value_heads = num_attention_heads
162
+
163
+ self.num_key_value_heads = num_key_value_heads
164
+ self.hidden_act = hidden_act
165
+ self.initializer_range = initializer_range
166
+ self.norm_eps = norm_eps
167
+ self.use_cache = use_cache
168
+ self.rope_theta = rope_theta
169
+ self.rope_scaling = rope_scaling
170
+ rope_percent = rope_percent or kwargs.get("rope_percentage", None)
171
+ self.rope_percent = rope_percent
172
+ self._rope_scaling_validation()
173
+ self.attention_bias = attention_bias
174
+ self.attention_dropout = attention_dropout
175
+ self.mlp_bias = mlp_bias
176
+
177
+ super().__init__(
178
+ pad_token_id=pad_token_id,
179
+ bos_token_id=bos_token_id,
180
+ eos_token_id=eos_token_id,
181
+ tie_word_embeddings=tie_word_embeddings,
182
+ **kwargs,
183
+ )
184
+
185
+ def _rope_scaling_validation(self):
186
+ """
187
+ Validate the `rope_scaling` configuration.
188
+ """
189
+ if self.rope_scaling is None:
190
+ return
191
+
192
+ if not isinstance(self.rope_scaling,
193
+ dict) or len(self.rope_scaling) != 2:
194
+ raise ValueError(
195
+ "`rope_scaling` must be a dictionary with two fields, "
196
+ f"`type` and `factor`, got {self.rope_scaling}")
197
+ rope_scaling_type = self.rope_scaling.get("type", None)
198
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
199
+ if rope_scaling_type is None or rope_scaling_type not in [
200
+ "linear", "dynamic"
201
+ ]:
202
+ raise ValueError(
203
+ "`rope_scaling`'s type field must be one of ['linear', "
204
+ f"'dynamic'], got {rope_scaling_type}")
205
+ if rope_scaling_factor is None or not isinstance(
206
+ rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
207
+ raise ValueError(
208
+ "`rope_scaling`'s factor field must be a float > 1, got "
209
+ f"{rope_scaling_factor}")