A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Align nemotron config with final HF state and fix lm-eval-small (#7611)
Michael Goin committed
Aug 16, 2024 at 18:56 UTC
44f26a94664584f20458ae0ddf1a826b2d79a13c
5 files changed
+29
-35
.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml
renamed
+4
-4
@@ -1,11 +1,11 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nvidia/Minitron-4B-Base -b auto -l 1000 -f 5 -t 1
2
-model_name: "nvidia/Minitron-4B-Base"
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m mgoin/Minitron-4B-Base-FP8 -b auto -l 1000 -f 5 -t 1
2
+model_name: "mgoin/Minitron-4B-Base-FP8"
3
tasks:
4
- name: "gsm8k"
5
metrics:
6
- name: "exact_match,strict-match"
7
- value: 0.252
7
+ value: 0.233
8
- name: "exact_match,flexible-extract"
9
- value: 0.252
9
+ value: 0.236
10
limit: 1000
11
num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
-1
@@ -4,7 +4,7 @@ Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
-Minitron-4B-Base.yaml
7
+Minitron-4B-Base-FP8.yaml
8
Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9
Qwen2-1.5B-Instruct-FP8W8.yaml
10
Meta-Llama-3-8B-QQQ.yaml
vllm/model_executor/layers/rotary_embedding.py
+3
-3
@@ -774,7 +774,7 @@ def get_rope(
774
is_neox_style: bool = True,
775
rope_scaling: Optional[Dict[str, Any]] = None,
776
dtype: Optional[torch.dtype] = None,
777
- rotary_percent: float = 1.0,
777
+ partial_rotary_factor: float = 1.0,
778
) -> RotaryEmbedding:
779
if dtype is None:
780
dtype = torch.get_default_dtype()
@@ -787,8 +787,8 @@ def get_rope(
787
rope_scaling_args = tuple(rope_scaling_tuple.items())
788
else:
789
rope_scaling_args = None
790
- if rotary_percent < 1.0:
791
- rotary_dim = int(rotary_dim * rotary_percent)
790
+ if partial_rotary_factor < 1.0:
791
+ rotary_dim = int(rotary_dim * partial_rotary_factor)
792
key = (head_size, rotary_dim, max_position, base, is_neox_style,
793
rope_scaling_args, dtype)
794
if key in _ROPE_DICT:
vllm/model_executor/models/nemotron.py
+3
-3
@@ -53,7 +53,7 @@ from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
53
# - There is no gate_proj, just up_proj
54
# - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm
55
# - Squared ReLU instead of SwiGLU
56
-# - Adds a rotary_percent to RoPE
56
+# - Adds a partial_rotary_factor to RoPE
57
58
59
def _cast_if_autocast_enabled(*args):
@@ -161,7 +161,7 @@ class NemotronAttention(nn.Module):
161
self.kv_size = self.num_kv_heads * self.head_dim
162
self.scaling = self.head_dim**-0.5
163
self.rope_theta = rope_theta
164
- self.rotary_percent = config.rope_percent
164
+ self.partial_rotary_factor = config.partial_rotary_factor
165
self.max_position_embeddings = max_position_embeddings
166
167
self.qkv_proj = QKVParallelLinear(
@@ -187,7 +187,7 @@ class NemotronAttention(nn.Module):
187
max_position=max_position_embeddings,
188
base=rope_theta,
189
rope_scaling=rope_scaling,
190
- rotary_percent=self.rotary_percent,
190
+ partial_rotary_factor=self.partial_rotary_factor,
191
)
192
self.attn = Attention(self.num_heads,
193
self.head_dim,
vllm/transformers_utils/configs/nemotron.py
+18
-24
@@ -35,20 +35,20 @@ class NemotronConfig(PretrainedConfig):
35
36
37
Args:
38
- vocab_size (`int`, *optional*, defaults to 32000):
38
+ vocab_size (`int`, *optional*, defaults to 256000):
39
Vocabulary size of the Nemotron model. Defines the number of
40
different tokens that can be represented by the
41
`inputs_ids` passed when calling [`NemotronModel`]
42
- hidden_size (`int`, *optional*, defaults to 4096):
42
+ hidden_size (`int`, *optional*, defaults to 6144):
43
Dimension of the hidden representations.
44
- intermediate_size (`int`, *optional*, defaults to 11008):
44
+ intermediate_size (`int`, *optional*, defaults to 24576):
45
Dimension of the MLP representations.
46
num_hidden_layers (`int`, *optional*, defaults to 32):
47
Number of hidden layers in the Transformer decoder.
48
- num_attention_heads (`int`, *optional*, defaults to 32):
48
+ num_attention_heads (`int`, *optional*, defaults to 48):
49
Number of attention heads for each attention layer in the
50
Transformer decoder.
51
- head_dim (`int`, *optional*, defaults to None):
51
+ head_dim (`int`, *optional*):
52
Projection weights dimension in multi-head attention. Set to
53
hidden_size // num_attention_heads if None
54
num_key_value_heads (`int`, *optional*):
@@ -63,16 +63,16 @@ class NemotronConfig(PretrainedConfig):
63
heads within that group. For more details checkout
64
[this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
65
is not specified, will default to `num_attention_heads`.
66
- hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
66
+ hidden_act (`str` or `function`, *optional*, defaults to `"relu2"`):
67
The non-linear activation function (function or string) in the
68
decoder.
69
- max_position_embeddings (`int`, *optional*, defaults to 2048):
69
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
70
The maximum sequence length that this model might ever be used
71
with.
72
- initializer_range (`float`, *optional*, defaults to 0.02):
72
+ initializer_range (`float`, *optional*, defaults to 0.0134):
73
The standard deviation of the truncated_normal_initializer for
74
initializing all weight matrices.
75
- norm_eps (`float`, *optional*, defaults to 1e-06):
75
+ norm_eps (`float`, *optional*, defaults to 1e-05):
76
The epsilon used by the normalization layers.
77
use_cache (`bool`, *optional*, defaults to `True`):
78
Whether or not the model should return the last key/values
@@ -80,21 +80,16 @@ class NemotronConfig(PretrainedConfig):
80
`config.is_decoder=True`.
81
pad_token_id (`int`, *optional*):
82
Padding token id.
83
- bos_token_id (`int`, *optional*, defaults to 1):
83
+ bos_token_id (`int`, *optional*, defaults to 2):
84
Beginning of stream token id.
85
- eos_token_id (`int`, *optional*, defaults to 2):
85
+ eos_token_id (`int`, *optional*, defaults to 3):
86
End of stream token id.
87
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88
Whether to tie weight embeddings
89
rope_theta (`float`, *optional*, defaults to 10000.0):
90
The base period of the RoPE embeddings.
91
- rope_scaling (`Dict`, *optional*):
92
- Dictionary containing the scaling configuration for the RoPE
93
- embeddings. Currently supports two scaling strategies: linear
94
- and dynamic. Their scaling factor must be a float greater than 1.
95
- The expected format is `{"type": strategy name,
96
- "factor": scaling factor}`. When using this flag, don't update
97
- `max_position_embeddings` to the expected new maximum.
91
+ partial_rotary_factor (`float`, *optional*, defaults to 0.5):
92
+ Percentage of the query and keys which will have rotary embedding.
93
attention_bias (`bool`, *optional*, defaults to `False`):
94
Whether to use a bias in the query, key, value and output
95
projection layers during self-attention.
@@ -106,13 +101,10 @@ class NemotronConfig(PretrainedConfig):
101
102
```python
103
>>> from transformers import NemotronModel, NemotronConfig
109
-
104
>>> # Initializing a Nemotron nemotron-15b style configuration
105
>>> configuration = NemotronConfig()
112
-
106
>>> # Initializing a model from the nemotron-15b style configuration
107
>>> model = NemotronModel(configuration)
115
-
108
>>> # Accessing the model configuration
109
>>> configuration = model.config
110
```"""
@@ -140,7 +132,7 @@ class NemotronConfig(PretrainedConfig):
132
tie_word_embeddings=False,
133
rope_theta=10000.0,
134
rope_scaling=None,
143
- rope_percent=0.5,
135
+ partial_rotary_factor=0.5,
136
attention_bias=False,
137
attention_dropout=0.0,
138
mlp_bias=False,
@@ -167,8 +159,10 @@ class NemotronConfig(PretrainedConfig):
159
self.use_cache = use_cache
160
self.rope_theta = rope_theta
161
self.rope_scaling = rope_scaling
170
- rope_percent = rope_percent or kwargs.get("rope_percentage", None)
171
- self.rope_percent = rope_percent
162
+ # for backward compatibility
163
+ partial_rotary_factor = kwargs.get("rope_percent", None) or kwargs.get(
164
+ "rope_percentage", None) or partial_rotary_factor
165
+ self.partial_rotary_factor = partial_rotary_factor
166
self._rope_scaling_validation()
167
self.attention_bias = attention_bias
168
self.attention_dropout = attention_dropout