@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07278c37dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Support Nemotron models (Nemotron-3, Nemotron-4, Minitron) (#6611)

Michael Goin committed Jul 26, 2024 at 14:33 UTC 07278c37ddd898d842bbddc382e4f67ac08dae35
9 files changed +776 -1
.buildkite/lm-eval-harness/configs/Minitron-4B-Base.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nvidia/Minitron-4B-Base -b auto -l 1000 -f 5 -t 1
2 +model_name: "nvidia/Minitron-4B-Base"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.252
8 + - name: "exact_match,flexible-extract"
9 + value: 0.252
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -4,5 +4,6 @@ Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7 +Minitron-4B-Base.yaml
8 Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9 Qwen2-1.5B-Instruct-FP8W8.yaml
vllm/model_executor/layers/activation.py
+16
@@ -159,6 +159,21 @@ class QuickGELU(CustomOp):
159 # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
160
161
162 +class ReLUSquaredActivation(CustomOp):
163 + """
164 + Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
165 + """
166 +
167 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
168 + """PyTorch-native implementation equivalent to forward()."""
169 + relu_applied = nn.functional.relu(x)
170 + squared = torch.square(relu_applied)
171 + return squared
172 +
173 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
174 + return self.forward_native(x)
175 +
176 +
177 class ScaledActivation(nn.Module):
178 """An activation function with post-scale parameters.
179
@@ -207,6 +222,7 @@ _ACTIVATION_REGISTRY = {
222 "gelu_new": NewGELU(),
223 "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
224 "relu": nn.ReLU(),
225 + "relu2": ReLUSquaredActivation(),
226 "quick_gelu": QuickGELU(),
227 }
228
vllm/model_executor/layers/rotary_embedding.py
+3
@@ -774,6 +774,7 @@ def get_rope(
774 is_neox_style: bool = True,
775 rope_scaling: Optional[Dict[str, Any]] = None,
776 dtype: Optional[torch.dtype] = None,
777 + rotary_percent: float = 1.0,
778 ) -> RotaryEmbedding:
779 if dtype is None:
780 dtype = torch.get_default_dtype()
@@ -786,6 +787,8 @@ def get_rope(
787 rope_scaling_args = tuple(rope_scaling_tuple.items())
788 else:
789 rope_scaling_args = None
790 + if rotary_percent < 1.0:
791 + rotary_dim = int(rotary_dim * rotary_percent)
792 key = (head_size, rotary_dim, max_position, base, is_neox_style,
793 rope_scaling_args, dtype)
794 if key in _ROPE_DICT:
vllm/model_executor/models/__init__.py
+1
@@ -51,6 +51,7 @@ _GENERATION_MODELS = {
51 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
52 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
53 "MiniCPMV": ("minicpmv", "MiniCPMV"),
54 + "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
55 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
56 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
57 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/nemotron.py new
+531
@@ -0,0 +1,531 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Nemotron model compatible with HuggingFace weights."""
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25 +
26 +import torch
27 +from torch import nn
28 +
29 +from vllm.attention import Attention, AttentionMetadata
30 +from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
32 +from vllm.model_executor.layers.activation import get_act_fn
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + QKVParallelLinear,
35 + RowParallelLinear)
36 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 +from vllm.model_executor.layers.quantization.base_config import (
38 + QuantizationConfig)
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.vocab_parallel_embedding import (
42 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 +from vllm.model_executor.model_loader.weight_utils import (
44 + default_weight_loader, maybe_remap_kv_scale_name)
45 +from vllm.model_executor.sampling_metadata import SamplingMetadata
46 +from vllm.sequence import IntermediateTensors, SamplerOutput
47 +from vllm.transformers_utils.configs import NemotronConfig
48 +
49 +from .interfaces import SupportsLoRA
50 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
51 +
52 +# The architecture is pretty similar to Llama, with these changes:
53 +# - There is no gate_proj, just up_proj
54 +# - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm
55 +# - Squared ReLU instead of SwiGLU
56 +# - Adds a rotary_percent to RoPE
57 +
58 +
59 +def _cast_if_autocast_enabled(*args):
60 + if not torch.is_autocast_enabled():
61 + return args
62 + else:
63 + return torch.cuda.amp.autocast_mode._cast(
64 + args, torch.get_autocast_gpu_dtype())
65 +
66 +
67 +class NemotronLayerNorm1P(nn.LayerNorm):
68 +
69 + def __init__(self,
70 + normalized_shape: Union[int, List[int], torch.Size],
71 + eps: float = 1e-5,
72 + elementwise_affine: bool = True,
73 + bias: bool = True,
74 + device=None,
75 + dtype=None):
76 + super().__init__(normalized_shape, eps, elementwise_affine, bias,
77 + device, dtype)
78 +
79 + def forward(
80 + self,
81 + x: torch.Tensor,
82 + residual: Optional[torch.Tensor] = None,
83 + ) -> torch.Tensor:
84 + if residual is not None:
85 + x = x + residual
86 + residual = x
87 + args = _cast_if_autocast_enabled(x, self.normalized_shape,
88 + self.weight + 1, self.bias, self.eps)
89 + with torch.cuda.amp.autocast(enabled=False):
90 + x = torch.nn.functional.layer_norm(*args)
91 + return x if residual is None else (x, residual)
92 +
93 +
94 +class NemotronMLP(nn.Module):
95 +
96 + def __init__(
97 + self,
98 + hidden_size: int,
99 + intermediate_size: int,
100 + hidden_act: str,
101 + quant_config: Optional[QuantizationConfig] = None,
102 + bias: bool = False,
103 + prefix: str = "",
104 + ) -> None:
105 + super().__init__()
106 + self.up_proj = ColumnParallelLinear(input_size=hidden_size,
107 + output_size=intermediate_size,
108 + bias=bias,
109 + quant_config=quant_config,
110 + prefix=f"{prefix}.up_proj")
111 + self.down_proj = RowParallelLinear(input_size=intermediate_size,
112 + output_size=hidden_size,
113 + bias=bias,
114 + quant_config=quant_config,
115 + prefix=f"{prefix}.down_proj")
116 + self.act_fn = get_act_fn(hidden_act)
117 +
118 + def forward(self, x):
119 + up, _ = self.up_proj(x)
120 + x = self.act_fn(up)
121 + x, _ = self.down_proj(x)
122 + return x
123 +
124 +
125 +class NemotronAttention(nn.Module):
126 +
127 + def __init__(
128 + self,
129 + config: NemotronConfig,
130 + hidden_size: int,
131 + num_heads: int,
132 + num_kv_heads: int,
133 + rope_theta: float = 10000,
134 + rope_scaling: Optional[Dict[str, Any]] = None,
135 + max_position_embeddings: int = 8192,
136 + quant_config: Optional[QuantizationConfig] = None,
137 + bias: bool = False,
138 + cache_config: Optional[CacheConfig] = None,
139 + prefix: str = "",
140 + ) -> None:
141 + super().__init__()
142 + self.hidden_size = hidden_size
143 + tp_size = get_tensor_model_parallel_world_size()
144 + self.total_num_heads = num_heads
145 + assert self.total_num_heads % tp_size == 0
146 + self.num_heads = self.total_num_heads // tp_size
147 + self.total_num_kv_heads = num_kv_heads
148 + if self.total_num_kv_heads >= tp_size:
149 + # Number of KV heads is greater than TP size, so we partition
150 + # the KV heads across multiple tensor parallel GPUs.
151 + assert self.total_num_kv_heads % tp_size == 0
152 + else:
153 + # Number of KV heads is less than TP size, so we replicate
154 + # the KV heads across multiple tensor parallel GPUs.
155 + assert tp_size % self.total_num_kv_heads == 0
156 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
157 + # MistralConfig has an optional head_dim introduced by Mistral-Nemo
158 + self.head_dim = getattr(config, "head_dim",
159 + self.hidden_size // self.total_num_heads)
160 + self.q_size = self.num_heads * self.head_dim
161 + self.kv_size = self.num_kv_heads * self.head_dim
162 + self.scaling = self.head_dim**-0.5
163 + self.rope_theta = rope_theta
164 + self.rotary_percent = config.rope_percent
165 + self.max_position_embeddings = max_position_embeddings
166 +
167 + self.qkv_proj = QKVParallelLinear(
168 + hidden_size=hidden_size,
169 + head_size=self.head_dim,
170 + total_num_heads=self.total_num_heads,
171 + total_num_kv_heads=self.total_num_kv_heads,
172 + bias=bias,
173 + quant_config=quant_config,
174 + prefix=f"{prefix}.qkv_proj",
175 + )
176 + self.o_proj = RowParallelLinear(
177 + input_size=self.total_num_heads * self.head_dim,
178 + output_size=hidden_size,
179 + bias=bias,
180 + quant_config=quant_config,
181 + prefix=f"{prefix}.o_proj",
182 + )
183 +
184 + self.rotary_emb = get_rope(
185 + self.head_dim,
186 + rotary_dim=self.head_dim,
187 + max_position=max_position_embeddings,
188 + base=rope_theta,
189 + rope_scaling=rope_scaling,
190 + rotary_percent=self.rotary_percent,
191 + )
192 + self.attn = Attention(self.num_heads,
193 + self.head_dim,
194 + self.scaling,
195 + num_kv_heads=self.num_kv_heads,
196 + cache_config=cache_config,
197 + quant_config=quant_config)
198 +
199 + def forward(
200 + self,
201 + positions: torch.Tensor,
202 + hidden_states: torch.Tensor,
203 + kv_cache: torch.Tensor,
204 + attn_metadata: AttentionMetadata,
205 + ) -> torch.Tensor:
206 + qkv, _ = self.qkv_proj(hidden_states)
207 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
208 + q, k = self.rotary_emb(positions, q, k)
209 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
210 + output, _ = self.o_proj(attn_output)
211 + return output
212 +
213 +
214 +class NemotronDecoderLayer(nn.Module):
215 +
216 + def __init__(
217 + self,
218 + config: NemotronConfig,
219 + cache_config: Optional[CacheConfig] = None,
220 + quant_config: Optional[QuantizationConfig] = None,
221 + prefix: str = "",
222 + ) -> None:
223 + super().__init__()
224 + self.hidden_size = config.hidden_size
225 + rope_theta = getattr(config, "rope_theta", 10000)
226 + rope_scaling = getattr(config, "rope_scaling", None)
227 + if rope_scaling is not None and getattr(
228 + config, "original_max_position_embeddings", None):
229 + rope_scaling["original_max_position_embeddings"] = (
230 + config.original_max_position_embeddings)
231 + max_position_embeddings = getattr(config, "max_position_embeddings",
232 + 8192)
233 + # Support abacusai/Smaug-72B-v0.1 with attention_bias
234 + # Support internlm/internlm-7b with bias
235 + attention_bias = getattr(config, "attention_bias", False) or getattr(
236 + config, "bias", False)
237 + self.self_attn = NemotronAttention(
238 + config=config,
239 + hidden_size=self.hidden_size,
240 + num_heads=config.num_attention_heads,
241 + num_kv_heads=getattr(config, "num_key_value_heads",
242 + config.num_attention_heads),
243 + rope_theta=rope_theta,
244 + rope_scaling=rope_scaling,
245 + max_position_embeddings=max_position_embeddings,
246 + quant_config=quant_config,
247 + bias=attention_bias,
248 + cache_config=cache_config,
249 + prefix=f"{prefix}.self_attn",
250 + )
251 + self.mlp = NemotronMLP(
252 + hidden_size=self.hidden_size,
253 + intermediate_size=config.intermediate_size,
254 + hidden_act=config.hidden_act,
255 + quant_config=quant_config,
256 + bias=getattr(config, "mlp_bias", False),
257 + prefix=f"{prefix}.mlp",
258 + )
259 + self.input_layernorm = NemotronLayerNorm1P(config.hidden_size,
260 + eps=config.norm_eps)
261 + self.post_attention_layernorm = NemotronLayerNorm1P(
262 + config.hidden_size, eps=config.norm_eps)
263 +
264 + def forward(
265 + self,
266 + positions: torch.Tensor,
267 + hidden_states: torch.Tensor,
268 + kv_cache: torch.Tensor,
269 + attn_metadata: AttentionMetadata,
270 + residual: Optional[torch.Tensor],
271 + ) -> Tuple[torch.Tensor, torch.Tensor]:
272 + # Self Attention
273 + if residual is None:
274 + residual = hidden_states
275 + hidden_states = self.input_layernorm(hidden_states)
276 + else:
277 + hidden_states, residual = self.input_layernorm(
278 + hidden_states, residual)
279 + hidden_states = self.self_attn(
280 + positions=positions,
281 + hidden_states=hidden_states,
282 + kv_cache=kv_cache,
283 + attn_metadata=attn_metadata,
284 + )
285 +
286 + # Fully Connected
287 + hidden_states, residual = self.post_attention_layernorm(
288 + hidden_states, residual)
289 + hidden_states = self.mlp(hidden_states)
290 + return hidden_states, residual
291 +
292 +
293 +class NemotronModel(nn.Module):
294 +
295 + def __init__(
296 + self,
297 + config: NemotronConfig,
298 + cache_config: Optional[CacheConfig] = None,
299 + quant_config: Optional[QuantizationConfig] = None,
300 + lora_config: Optional[LoRAConfig] = None,
301 + prefix: str = "",
302 + ) -> None:
303 + super().__init__()
304 + self.config = config
305 + self.padding_idx = config.pad_token_id
306 + lora_vocab = (lora_config.lora_extra_vocab_size *
307 + (lora_config.max_loras or 1)) if lora_config else 0
308 + self.vocab_size = config.vocab_size + lora_vocab
309 + self.org_vocab_size = config.vocab_size
310 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
311 + and get_pp_group().is_last_rank):
312 + self.embed_tokens = VocabParallelEmbedding(
313 + self.vocab_size,
314 + config.hidden_size,
315 + org_num_embeddings=config.vocab_size,
316 + )
317 + else:
318 + self.embed_tokens = PPMissingLayer()
319 + self.start_layer, self.end_layer, self.layers = make_layers(
320 + config.num_hidden_layers,
321 + lambda prefix: NemotronDecoderLayer(config=config,
322 + cache_config=cache_config,
323 + quant_config=quant_config,
324 + prefix=prefix),
325 + prefix=f"{prefix}.layers")
326 + if get_pp_group().is_last_rank:
327 + self.norm = NemotronLayerNorm1P(config.hidden_size,
328 + eps=config.norm_eps)
329 + else:
330 + self.norm = PPMissingLayer()
331 +
332 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
333 + return self.embed_tokens(input_ids)
334 +
335 + def forward(
336 + self,
337 + input_ids: Optional[torch.Tensor],
338 + positions: torch.Tensor,
339 + kv_caches: List[torch.Tensor],
340 + attn_metadata: AttentionMetadata,
341 + intermediate_tensors: Optional[IntermediateTensors],
342 + inputs_embeds: Optional[torch.Tensor] = None,
343 + ) -> Union[torch.Tensor, IntermediateTensors]:
344 + if get_pp_group().is_first_rank:
345 + if inputs_embeds is not None:
346 + hidden_states = inputs_embeds
347 + else:
348 + hidden_states = self.get_input_embeddings(input_ids)
349 + residual = None
350 + else:
351 + assert intermediate_tensors is not None
352 + hidden_states = intermediate_tensors["hidden_states"]
353 + residual = intermediate_tensors["residual"]
354 +
355 + for i in range(self.start_layer, self.end_layer):
356 + layer = self.layers[i]
357 + hidden_states, residual = layer(
358 + positions,
359 + hidden_states,
360 + kv_caches[i - self.start_layer],
361 + attn_metadata,
362 + residual,
363 + )
364 +
365 + if not get_pp_group().is_last_rank:
366 + return IntermediateTensors({
367 + "hidden_states": hidden_states,
368 + "residual": residual
369 + })
370 +
371 + hidden_states, _ = self.norm(hidden_states, residual)
372 + return hidden_states
373 +
374 +
375 +class NemotronForCausalLM(nn.Module, SupportsLoRA):
376 + packed_modules_mapping = {
377 + "qkv_proj": [
378 + "q_proj",
379 + "k_proj",
380 + "v_proj",
381 + ],
382 + }
383 +
384 + # LoRA specific attributes
385 + supported_lora_modules = [
386 + "qkv_proj", "o_proj", "up_proj", "down_proj", "embed_tokens", "lm_head"
387 + ]
388 + embedding_modules = {
389 + "embed_tokens": "input_embeddings",
390 + "lm_head": "output_embeddings",
391 + }
392 + embedding_padding_modules = ["lm_head"]
393 + bitsandbytes_stacked_params_mapping = {
394 + # shard_name, weight_name, index
395 + "q_proj": ("qkv_proj", 0),
396 + "k_proj": ("qkv_proj", 1),
397 + "v_proj": ("qkv_proj", 2),
398 + }
399 +
400 + def __init__(
401 + self,
402 + config: NemotronConfig,
403 + cache_config: Optional[CacheConfig] = None,
404 + quant_config: Optional[QuantizationConfig] = None,
405 + lora_config: Optional[LoRAConfig] = None,
406 + ) -> None:
407 + super().__init__()
408 +
409 + assert isinstance(config, NemotronConfig)
410 +
411 + self.config = config
412 + self.lora_config = lora_config
413 +
414 + self.model = NemotronModel(config,
415 + cache_config,
416 + quant_config,
417 + lora_config=lora_config,
418 + prefix="model")
419 + if get_pp_group().is_last_rank:
420 + self.unpadded_vocab_size = config.vocab_size
421 + if lora_config:
422 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
423 + self.lm_head = ParallelLMHead(
424 + self.unpadded_vocab_size,
425 + config.hidden_size,
426 + org_num_embeddings=config.vocab_size,
427 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
428 + # We need bigger padding if using lora for kernel
429 + # compatibility
430 + if not lora_config else lora_config.lora_vocab_padding_size,
431 + quant_config=quant_config,
432 + )
433 + if config.tie_word_embeddings:
434 + self.lm_head.weight = self.model.embed_tokens.weight
435 +
436 + logit_scale = getattr(config, "logit_scale", 1.0)
437 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
438 + config.vocab_size,
439 + logit_scale)
440 + self.sampler = Sampler()
441 + else:
442 + self.lm_head = PPMissingLayer()
443 +
444 + def forward(
445 + self,
446 + input_ids: torch.Tensor,
447 + positions: torch.Tensor,
448 + kv_caches: List[torch.Tensor],
449 + attn_metadata: AttentionMetadata,
450 + intermediate_tensors: Optional[IntermediateTensors] = None,
451 + ) -> Union[torch.Tensor, IntermediateTensors]:
452 + model_output = self.model(input_ids, positions, kv_caches,
453 + attn_metadata, intermediate_tensors)
454 + return model_output
455 +
456 + def compute_logits(self, hidden_states: torch.Tensor,
457 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
458 + logits = self.logits_processor(self.lm_head, hidden_states,
459 + sampling_metadata)
460 + return logits
461 +
462 + def sample(
463 + self,
464 + logits: torch.Tensor,
465 + sampling_metadata: SamplingMetadata,
466 + ) -> Optional[SamplerOutput]:
467 + next_tokens = self.sampler(logits, sampling_metadata)
468 + return next_tokens
469 +
470 + def make_empty_intermediate_tensors(
471 + self, batch_size: int, dtype: torch.dtype,
472 + device: torch.device) -> IntermediateTensors:
473 + return IntermediateTensors({
474 + "hidden_states":
475 + torch.zeros((batch_size, self.config.hidden_size),
476 + dtype=dtype,
477 + device=device),
478 + "residual":
479 + torch.zeros((batch_size, self.config.hidden_size),
480 + dtype=dtype,
481 + device=device),
482 + })
483 +
484 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
485 + stacked_params_mapping = [
486 + # (param_name, shard_name, shard_id)
487 + (".qkv_proj", ".q_proj", "q"),
488 + (".qkv_proj", ".k_proj", "k"),
489 + (".qkv_proj", ".v_proj", "v"),
490 + ]
491 + params_dict = dict(self.named_parameters())
492 + for name, loaded_weight in weights:
493 + if "rotary_emb.inv_freq" in name:
494 + continue
495 + if ("rotary_emb.cos_cached" in name
496 + or "rotary_emb.sin_cached" in name):
497 + # Models trained using ColossalAI may include these tensors in
498 + # the checkpoint. Skip them.
499 + continue
500 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
501 + if weight_name not in name:
502 + continue
503 + name = name.replace(weight_name, param_name)
504 + # Skip loading extra bias for GPTQ models.
505 + if name.endswith(".bias") and name not in params_dict:
506 + continue
507 +
508 + if is_pp_missing_parameter(name, self):
509 + continue
510 +
511 + param = params_dict[name]
512 + weight_loader = param.weight_loader
513 + weight_loader(param, loaded_weight, shard_id)
514 +
515 + break
516 + else:
517 + # Skip loading extra bias for GPTQ models.
518 + if name.endswith(".bias") and name not in params_dict:
519 + continue
520 + # Remapping the name of FP8 kv-scale.
521 + name = maybe_remap_kv_scale_name(name, params_dict)
522 + if name is None:
523 + continue
524 +
525 + if is_pp_missing_parameter(name, self):
526 + continue
527 +
528 + param = params_dict[name]
529 + weight_loader = getattr(param, "weight_loader",
530 + default_weight_loader)
531 + weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+2 -1
@@ -8,7 +8,7 @@ from vllm.logger import init_logger
8 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9 JAISConfig, MedusaConfig,
10 MLPSpeculatorConfig, MPTConfig,
11 - RWConfig)
11 + NemotronConfig, RWConfig)
12
13 if VLLM_USE_MODELSCOPE:
14 from modelscope import AutoConfig
@@ -26,6 +26,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
26 "jais": JAISConfig,
27 "mlp_speculator": MLPSpeculatorConfig,
28 "medusa": MedusaConfig,
29 + "nemotron": NemotronConfig,
30 }
31
32 for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -8,6 +8,7 @@ from vllm.transformers_utils.configs.jais import JAISConfig
8 from vllm.transformers_utils.configs.medusa import MedusaConfig
9 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
10 from vllm.transformers_utils.configs.mpt import MPTConfig
11 +from vllm.transformers_utils.configs.nemotron import NemotronConfig
12
13 __all__ = [
14 "ChatGLMConfig",
@@ -17,4 +18,5 @@ __all__ = [
18 "JAISConfig",
19 "MedusaConfig",
20 "MLPSpeculatorConfig",
21 + "NemotronConfig",
22 ]
vllm/transformers_utils/configs/nemotron.py new
+209
@@ -0,0 +1,209 @@
1 +# coding=utf-8
2 +# Copyright 2024 HuggingFace Inc. team. All rights reserved.
3 +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
4 +#
5 +# Licensed under the Apache License, Version 2.0 (the "License");
6 +# you may not use this file except in compliance with the License.
7 +# You may obtain a copy of the License at
8 +#
9 +# http://www.apache.org/licenses/LICENSE-2.0
10 +#
11 +# Unless required by applicable law or agreed to in writing, software
12 +# distributed under the License is distributed on an "AS IS" BASIS,
13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 +# See the License for the specific language governing permissions and
15 +# limitations under the License.
16 +"""Nemotron model configuration"""
17 +
18 +from transformers import PretrainedConfig
19 +from transformers.utils import logging
20 +
21 +logger = logging.get_logger(__name__)
22 +
23 +
24 +class NemotronConfig(PretrainedConfig):
25 + r"""
26 + This is the configuration class to store the configuration of a
27 + [`NemotronModel`]. It is used to instantiate an Nemotron model
28 + according to the specified arguments, defining the model architecture.
29 + Instantiating a configuration with the defaults will yield a similar
30 + configuration to that of the Nemotron-8B.
31 +
32 + Configuration objects inherit from [`PretrainedConfig`] and can be
33 + used to control the model outputs. Read the documentation from
34 + [`PretrainedConfig`] for more information.
35 +
36 +
37 + Args:
38 + vocab_size (`int`, *optional*, defaults to 32000):
39 + Vocabulary size of the Nemotron model. Defines the number of
40 + different tokens that can be represented by the
41 + `inputs_ids` passed when calling [`NemotronModel`]
42 + hidden_size (`int`, *optional*, defaults to 4096):
43 + Dimension of the hidden representations.
44 + intermediate_size (`int`, *optional*, defaults to 11008):
45 + Dimension of the MLP representations.
46 + num_hidden_layers (`int`, *optional*, defaults to 32):
47 + Number of hidden layers in the Transformer decoder.
48 + num_attention_heads (`int`, *optional*, defaults to 32):
49 + Number of attention heads for each attention layer in the
50 + Transformer decoder.
51 + head_dim (`int`, *optional*, defaults to None):
52 + Projection weights dimension in multi-head attention. Set to
53 + hidden_size // num_attention_heads if None
54 + num_key_value_heads (`int`, *optional*):
55 + This is the number of key_value heads that should be used to
56 + implement Grouped Query Attention. If
57 + `num_key_value_heads=num_attention_heads`, the model will use
58 + Multi Head Attention (MHA), if
59 + `num_key_value_heads=1 the model will use Multi Query Attention
60 + (MQA) otherwise GQA is used. When converting a multi-head
61 + checkpoint to a GQA checkpoint, each group key and value
62 + head should be constructed by meanpooling all the original
63 + heads within that group. For more details checkout
64 + [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
65 + is not specified, will default to `num_attention_heads`.
66 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
67 + The non-linear activation function (function or string) in the
68 + decoder.
69 + max_position_embeddings (`int`, *optional*, defaults to 2048):
70 + The maximum sequence length that this model might ever be used
71 + with.
72 + initializer_range (`float`, *optional*, defaults to 0.02):
73 + The standard deviation of the truncated_normal_initializer for
74 + initializing all weight matrices.
75 + norm_eps (`float`, *optional*, defaults to 1e-06):
76 + The epsilon used by the normalization layers.
77 + use_cache (`bool`, *optional*, defaults to `True`):
78 + Whether or not the model should return the last key/values
79 + attentions (not used by all models). Only relevant if
80 + `config.is_decoder=True`.
81 + pad_token_id (`int`, *optional*):
82 + Padding token id.
83 + bos_token_id (`int`, *optional*, defaults to 1):
84 + Beginning of stream token id.
85 + eos_token_id (`int`, *optional*, defaults to 2):
86 + End of stream token id.
87 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88 + Whether to tie weight embeddings
89 + rope_theta (`float`, *optional*, defaults to 10000.0):
90 + The base period of the RoPE embeddings.
91 + rope_scaling (`Dict`, *optional*):
92 + Dictionary containing the scaling configuration for the RoPE
93 + embeddings. Currently supports two scaling strategies: linear
94 + and dynamic. Their scaling factor must be a float greater than 1.
95 + The expected format is `{"type": strategy name,
96 + "factor": scaling factor}`. When using this flag, don't update
97 + `max_position_embeddings` to the expected new maximum.
98 + attention_bias (`bool`, *optional*, defaults to `False`):
99 + Whether to use a bias in the query, key, value and output
100 + projection layers during self-attention.
101 + attention_dropout (`float`, *optional*, defaults to 0.0):
102 + The dropout ratio for the attention probabilities.
103 + mlp_bias (`bool`, *optional*, defaults to `False`):
104 + Whether to use a bias in up_proj and down_proj layers in the MLP
105 + layers.
106 +
107 + ```python
108 + >>> from transformers import NemotronModel, NemotronConfig
109 +
110 + >>> # Initializing a Nemotron nemotron-15b style configuration
111 + >>> configuration = NemotronConfig()
112 +
113 + >>> # Initializing a model from the nemotron-15b style configuration
114 + >>> model = NemotronModel(configuration)
115 +
116 + >>> # Accessing the model configuration
117 + >>> configuration = model.config
118 + ```"""
119 +
120 + model_type = "nemotron"
121 + keys_to_ignore_at_inference = ["past_key_values"]
122 +
123 + def __init__(
124 + self,
125 + vocab_size=256000,
126 + hidden_size=6144,
127 + intermediate_size=24576,
128 + num_hidden_layers=32,
129 + num_attention_heads=48,
130 + head_dim=None,
131 + num_key_value_heads=None,
132 + hidden_act="relu2",
133 + max_position_embeddings=4096,
134 + initializer_range=0.0134,
135 + norm_eps=1e-5,
136 + use_cache=True,
137 + pad_token_id=None,
138 + bos_token_id=2,
139 + eos_token_id=3,
140 + tie_word_embeddings=False,
141 + rope_theta=10000.0,
142 + rope_scaling=None,
143 + rope_percent=0.5,
144 + attention_bias=False,
145 + attention_dropout=0.0,
146 + mlp_bias=False,
147 + **kwargs,
148 + ):
149 + self.vocab_size = vocab_size
150 + self.max_position_embeddings = max_position_embeddings
151 + self.hidden_size = hidden_size
152 + self.intermediate_size = intermediate_size
153 + self.num_hidden_layers = num_hidden_layers
154 + self.num_attention_heads = num_attention_heads
155 + head_dim = head_dim or kwargs.get("kv_channels", None)
156 + self.head_dim = head_dim if head_dim is not None else (
157 + hidden_size // num_attention_heads)
158 +
159 + # for backward compatibility
160 + if num_key_value_heads is None:
161 + num_key_value_heads = num_attention_heads
162 +
163 + self.num_key_value_heads = num_key_value_heads
164 + self.hidden_act = hidden_act
165 + self.initializer_range = initializer_range
166 + self.norm_eps = norm_eps
167 + self.use_cache = use_cache
168 + self.rope_theta = rope_theta
169 + self.rope_scaling = rope_scaling
170 + rope_percent = rope_percent or kwargs.get("rope_percentage", None)
171 + self.rope_percent = rope_percent
172 + self._rope_scaling_validation()
173 + self.attention_bias = attention_bias
174 + self.attention_dropout = attention_dropout
175 + self.mlp_bias = mlp_bias
176 +
177 + super().__init__(
178 + pad_token_id=pad_token_id,
179 + bos_token_id=bos_token_id,
180 + eos_token_id=eos_token_id,
181 + tie_word_embeddings=tie_word_embeddings,
182 + **kwargs,
183 + )
184 +
185 + def _rope_scaling_validation(self):
186 + """
187 + Validate the `rope_scaling` configuration.
188 + """
189 + if self.rope_scaling is None:
190 + return
191 +
192 + if not isinstance(self.rope_scaling,
193 + dict) or len(self.rope_scaling) != 2:
194 + raise ValueError(
195 + "`rope_scaling` must be a dictionary with two fields, "
196 + f"`type` and `factor`, got {self.rope_scaling}")
197 + rope_scaling_type = self.rope_scaling.get("type", None)
198 + rope_scaling_factor = self.rope_scaling.get("factor", None)
199 + if rope_scaling_type is None or rope_scaling_type not in [
200 + "linear", "dynamic"
201 + ]:
202 + raise ValueError(
203 + "`rope_scaling`'s type field must be one of ['linear', "
204 + f"'dynamic'], got {rope_scaling_type}")
205 + if rope_scaling_factor is None or not isinstance(
206 + rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
207 + raise ValueError(
208 + "`rope_scaling`'s factor field must be a float > 1, got "
209 + f"{rope_scaling_factor}")