@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 44f26a9466
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Align nemotron config with final HF state and fix lm-eval-small (#7611)

Michael Goin committed Aug 16, 2024 at 18:56 UTC 44f26a94664584f20458ae0ddf1a826b2d79a13c
5 files changed +29 -35
.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml renamed
+4 -4
@@ -1,11 +1,11 @@
1 -# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nvidia/Minitron-4B-Base -b auto -l 1000 -f 5 -t 1
2 -model_name: "nvidia/Minitron-4B-Base"
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m mgoin/Minitron-4B-Base-FP8 -b auto -l 1000 -f 5 -t 1
2 +model_name: "mgoin/Minitron-4B-Base-FP8"
3 tasks:
4 - name: "gsm8k"
5 metrics:
6 - name: "exact_match,strict-match"
7 - value: 0.252
7 + value: 0.233
8 - name: "exact_match,flexible-extract"
9 - value: 0.252
9 + value: 0.236
10 limit: 1000
11 num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1 -1
@@ -4,7 +4,7 @@ Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7 -Minitron-4B-Base.yaml
7 +Minitron-4B-Base-FP8.yaml
8 Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9 Qwen2-1.5B-Instruct-FP8W8.yaml
10 Meta-Llama-3-8B-QQQ.yaml
vllm/model_executor/layers/rotary_embedding.py
+3 -3
@@ -774,7 +774,7 @@ def get_rope(
774 is_neox_style: bool = True,
775 rope_scaling: Optional[Dict[str, Any]] = None,
776 dtype: Optional[torch.dtype] = None,
777 - rotary_percent: float = 1.0,
777 + partial_rotary_factor: float = 1.0,
778 ) -> RotaryEmbedding:
779 if dtype is None:
780 dtype = torch.get_default_dtype()
@@ -787,8 +787,8 @@ def get_rope(
787 rope_scaling_args = tuple(rope_scaling_tuple.items())
788 else:
789 rope_scaling_args = None
790 - if rotary_percent < 1.0:
791 - rotary_dim = int(rotary_dim * rotary_percent)
790 + if partial_rotary_factor < 1.0:
791 + rotary_dim = int(rotary_dim * partial_rotary_factor)
792 key = (head_size, rotary_dim, max_position, base, is_neox_style,
793 rope_scaling_args, dtype)
794 if key in _ROPE_DICT:
vllm/model_executor/models/nemotron.py
+3 -3
@@ -53,7 +53,7 @@ from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
53 # - There is no gate_proj, just up_proj
54 # - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm
55 # - Squared ReLU instead of SwiGLU
56 -# - Adds a rotary_percent to RoPE
56 +# - Adds a partial_rotary_factor to RoPE
57
58
59 def _cast_if_autocast_enabled(*args):
@@ -161,7 +161,7 @@ class NemotronAttention(nn.Module):
161 self.kv_size = self.num_kv_heads * self.head_dim
162 self.scaling = self.head_dim**-0.5
163 self.rope_theta = rope_theta
164 - self.rotary_percent = config.rope_percent
164 + self.partial_rotary_factor = config.partial_rotary_factor
165 self.max_position_embeddings = max_position_embeddings
166
167 self.qkv_proj = QKVParallelLinear(
@@ -187,7 +187,7 @@ class NemotronAttention(nn.Module):
187 max_position=max_position_embeddings,
188 base=rope_theta,
189 rope_scaling=rope_scaling,
190 - rotary_percent=self.rotary_percent,
190 + partial_rotary_factor=self.partial_rotary_factor,
191 )
192 self.attn = Attention(self.num_heads,
193 self.head_dim,
vllm/transformers_utils/configs/nemotron.py
+18 -24
@@ -35,20 +35,20 @@ class NemotronConfig(PretrainedConfig):
35
36
37 Args:
38 - vocab_size (`int`, *optional*, defaults to 32000):
38 + vocab_size (`int`, *optional*, defaults to 256000):
39 Vocabulary size of the Nemotron model. Defines the number of
40 different tokens that can be represented by the
41 `inputs_ids` passed when calling [`NemotronModel`]
42 - hidden_size (`int`, *optional*, defaults to 4096):
42 + hidden_size (`int`, *optional*, defaults to 6144):
43 Dimension of the hidden representations.
44 - intermediate_size (`int`, *optional*, defaults to 11008):
44 + intermediate_size (`int`, *optional*, defaults to 24576):
45 Dimension of the MLP representations.
46 num_hidden_layers (`int`, *optional*, defaults to 32):
47 Number of hidden layers in the Transformer decoder.
48 - num_attention_heads (`int`, *optional*, defaults to 32):
48 + num_attention_heads (`int`, *optional*, defaults to 48):
49 Number of attention heads for each attention layer in the
50 Transformer decoder.
51 - head_dim (`int`, *optional*, defaults to None):
51 + head_dim (`int`, *optional*):
52 Projection weights dimension in multi-head attention. Set to
53 hidden_size // num_attention_heads if None
54 num_key_value_heads (`int`, *optional*):
@@ -63,16 +63,16 @@ class NemotronConfig(PretrainedConfig):
63 heads within that group. For more details checkout
64 [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
65 is not specified, will default to `num_attention_heads`.
66 - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
66 + hidden_act (`str` or `function`, *optional*, defaults to `"relu2"`):
67 The non-linear activation function (function or string) in the
68 decoder.
69 - max_position_embeddings (`int`, *optional*, defaults to 2048):
69 + max_position_embeddings (`int`, *optional*, defaults to 4096):
70 The maximum sequence length that this model might ever be used
71 with.
72 - initializer_range (`float`, *optional*, defaults to 0.02):
72 + initializer_range (`float`, *optional*, defaults to 0.0134):
73 The standard deviation of the truncated_normal_initializer for
74 initializing all weight matrices.
75 - norm_eps (`float`, *optional*, defaults to 1e-06):
75 + norm_eps (`float`, *optional*, defaults to 1e-05):
76 The epsilon used by the normalization layers.
77 use_cache (`bool`, *optional*, defaults to `True`):
78 Whether or not the model should return the last key/values
@@ -80,21 +80,16 @@ class NemotronConfig(PretrainedConfig):
80 `config.is_decoder=True`.
81 pad_token_id (`int`, *optional*):
82 Padding token id.
83 - bos_token_id (`int`, *optional*, defaults to 1):
83 + bos_token_id (`int`, *optional*, defaults to 2):
84 Beginning of stream token id.
85 - eos_token_id (`int`, *optional*, defaults to 2):
85 + eos_token_id (`int`, *optional*, defaults to 3):
86 End of stream token id.
87 tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88 Whether to tie weight embeddings
89 rope_theta (`float`, *optional*, defaults to 10000.0):
90 The base period of the RoPE embeddings.
91 - rope_scaling (`Dict`, *optional*):
92 - Dictionary containing the scaling configuration for the RoPE
93 - embeddings. Currently supports two scaling strategies: linear
94 - and dynamic. Their scaling factor must be a float greater than 1.
95 - The expected format is `{"type": strategy name,
96 - "factor": scaling factor}`. When using this flag, don't update
97 - `max_position_embeddings` to the expected new maximum.
91 + partial_rotary_factor (`float`, *optional*, defaults to 0.5):
92 + Percentage of the query and keys which will have rotary embedding.
93 attention_bias (`bool`, *optional*, defaults to `False`):
94 Whether to use a bias in the query, key, value and output
95 projection layers during self-attention.
@@ -106,13 +101,10 @@ class NemotronConfig(PretrainedConfig):
101
102 ```python
103 >>> from transformers import NemotronModel, NemotronConfig
109 -
104 >>> # Initializing a Nemotron nemotron-15b style configuration
105 >>> configuration = NemotronConfig()
112 -
106 >>> # Initializing a model from the nemotron-15b style configuration
107 >>> model = NemotronModel(configuration)
115 -
108 >>> # Accessing the model configuration
109 >>> configuration = model.config
110 ```"""
@@ -140,7 +132,7 @@ class NemotronConfig(PretrainedConfig):
132 tie_word_embeddings=False,
133 rope_theta=10000.0,
134 rope_scaling=None,
143 - rope_percent=0.5,
135 + partial_rotary_factor=0.5,
136 attention_bias=False,
137 attention_dropout=0.0,
138 mlp_bias=False,
@@ -167,8 +159,10 @@ class NemotronConfig(PretrainedConfig):
159 self.use_cache = use_cache
160 self.rope_theta = rope_theta
161 self.rope_scaling = rope_scaling
170 - rope_percent = rope_percent or kwargs.get("rope_percentage", None)
171 - self.rope_percent = rope_percent
162 + # for backward compatibility
163 + partial_rotary_factor = kwargs.get("rope_percent", None) or kwargs.get(
164 + "rope_percentage", None) or partial_rotary_factor
165 + self.partial_rotary_factor = partial_rotary_factor
166 self._rope_scaling_validation()
167 self.attention_bias = attention_bias
168 self.attention_dropout = attention_dropout