A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[bugfix] interleaving sliding window for cohere2 model (#11583)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 29, 2024 at 00:55 UTC
328841d00294fb8226f0368cc380350b3d671d77
7 files changed
+206
-13
docs/source/models/supported_models.md
+1
-1
@@ -112,7 +112,7 @@ See [this page](#generative-models) for more information on how to use generativ
112
- :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
113
- ✅︎
114
- ✅︎
115
- * - :code:`CohereForCausalLM`,:code:`Cohere2ForCausalLM`
115
+ * - :code:`CohereForCausalLM`, :code:`Cohere2ForCausalLM`
116
- Command-R
117
- :code:`CohereForAI/c4ai-command-r-v01`, :code:`CohereForAI/c4ai-command-r7b-12-2024`, etc.
118
- ✅︎
tests/models/test_initialization.py
-4
@@ -1,7 +1,6 @@
1
from unittest.mock import patch
2
3
import pytest
4
-import transformers
4
from transformers import PretrainedConfig
5
6
from vllm import LLM
@@ -12,9 +11,6 @@ from .registry import HF_EXAMPLE_MODELS
11
@pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
12
def test_can_initialize(model_arch):
13
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
15
- if (model_arch == "Cohere2ForCausalLM"
16
- and transformers.__version__ < "4.48.0"):
17
- pytest.skip(reason="Model introduced in HF >= 4.48.0")
14
if not model_info.is_available_online:
15
pytest.skip("Model is not available online")
16
vllm/config.py
+1
-1
@@ -301,7 +301,7 @@ class ModelConfig:
301
sliding_window = getattr(self.hf_text_config, "sliding_window", None)
302
has_interleaved_attention = (sliding_window is not None) and (
303
isinstance(sliding_window, list) or
304
- (self.hf_text_config.model_type in ["gemma2"]))
304
+ (self.hf_text_config.model_type in ["gemma2", "cohere2"]))
305
306
if (not self.disable_sliding_window and has_interleaved_attention):
307
if envs.VLLM_ATTENTION_BACKEND == "XFORMERS":
vllm/model_executor/models/commandr.py
+6
-4
@@ -172,16 +172,18 @@ class CohereAttention(nn.Module):
172
is_neox_style=False,
173
)
174
175
- sliding_window = getattr(config, "sliding_window", None)
176
- # Model v2 has sliding windows, v1 does not
177
- self.v1 = sliding_window is None
175
+ # Model v2 has interleaved sliding windows, v1 does not
176
+ interleaved_sliding_window = getattr(config,
177
+ "interleaved_sliding_window",
178
+ None)
179
+ self.v1 = interleaved_sliding_window is None
180
181
layer_idx = extract_layer_index(prefix)
182
layer_has_sliding_window = (
183
getattr(config, "sliding_window_pattern", False)
184
and (layer_idx + 1) % self.config.sliding_window_pattern != 0)
185
184
- self.sliding_window = (sliding_window
186
+ self.sliding_window = (interleaved_sliding_window
187
if layer_has_sliding_window else None)
188
189
self.attn = Attention(self.num_heads,
vllm/transformers_utils/config.py
+4
-3
@@ -22,9 +22,9 @@ from vllm.envs import VLLM_USE_MODELSCOPE
22
from vllm.logger import init_logger
23
# yapf conflicts with isort for this block
24
# yapf: disable
25
-from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
26
- EAGLEConfig, ExaoneConfig,
27
- H2OVLChatConfig,
25
+from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26
+ DbrxConfig, EAGLEConfig,
27
+ ExaoneConfig, H2OVLChatConfig,
28
InternVLChatConfig, JAISConfig,
29
MedusaConfig, MllamaConfig,
30
MLPSpeculatorConfig, MPTConfig,
@@ -52,6 +52,7 @@ _CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
52
53
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
54
"chatglm": ChatGLMConfig,
55
+ "cohere2": Cohere2Config,
56
"dbrx": DbrxConfig,
57
"mpt": MPTConfig,
58
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,4 +1,5 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
+from vllm.transformers_utils.configs.cohere2 import Cohere2Config
3
from vllm.transformers_utils.configs.dbrx import DbrxConfig
4
from vllm.transformers_utils.configs.eagle import EAGLEConfig
5
from vllm.transformers_utils.configs.exaone import ExaoneConfig
@@ -22,6 +23,7 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig
23
24
__all__ = [
25
"ChatGLMConfig",
26
+ "Cohere2Config",
27
"DbrxConfig",
28
"MPTConfig",
29
"RWConfig",
vllm/transformers_utils/configs/cohere2.py
new
+192
@@ -0,0 +1,192 @@
1
+# ruff: noqa
2
+
3
+# Adapted from
4
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2/configuration_cohere2.py
5
+from transformers import PretrainedConfig
6
+from transformers.modeling_rope_utils import rope_config_validation
7
+
8
+
9
+class Cohere2Config(PretrainedConfig):
10
+ r"""
11
+ This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
12
+ model according to the specified arguments, defining the model architecture.
13
+
14
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
15
+ documentation from [`PretrainedConfig`] for more information. Instantiating a configuration
16
+ with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.
17
+
18
+
19
+ Args:
20
+ vocab_size (`int`, *optional*, defaults to 256000):
21
+ Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
22
+ `inputs_ids` passed when calling [`CohereModel`]
23
+ hidden_size (`int`, *optional*, defaults to 8192):
24
+ Dimension of the hidden representations.
25
+ intermediate_size (`int`, *optional*, defaults to 22528):
26
+ Dimension of the MLP representations.
27
+ logit_scale (`float`, *optional*, defaults to 0.0625):
28
+ The scaling factor for the output logits.
29
+ num_hidden_layers (`int`, *optional*, defaults to 40):
30
+ Number of hidden layers in the Transformer decoder.
31
+ num_attention_heads (`int`, *optional*, defaults to 64):
32
+ Number of attention heads for each attention layer in the Transformer decoder.
33
+ num_key_value_heads (`int`, *optional*):
34
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
35
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
36
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
37
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
38
+ by meanpooling all the original heads within that group. For more details checkout [this
39
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
40
+ `num_attention_heads`.
41
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
42
+ The non-linear activation function (function or string) in the decoder.
43
+ max_position_embeddings (`int`, *optional*, defaults to 8192):
44
+ The maximum sequence length that this model might ever be used with.
45
+ initializer_range (`float`, *optional*, defaults to 0.02):
46
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
47
+ layer_norm_eps (`float`, *optional*, defaults to 1e-05):
48
+ The epsilon used by the layer normalization.
49
+ use_cache (`bool`, *optional*, defaults to `True`):
50
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
51
+ relevant if `config.is_decoder=True`.
52
+ pad_token_id (`int`, *optional*, defaults to 0):
53
+ Padding token id.
54
+ bos_token_id (`int`, *optional*, defaults to 5):
55
+ Beginning of stream token id.
56
+ eos_token_id (`int`, *optional*, defaults to 255001):
57
+ End of stream token id.
58
+ tie_word_embeddings (`bool`, *optional*, defaults to `True`):
59
+ Whether to tie weight embeddings
60
+ rope_theta (`float`, *optional*, defaults to 10000.0):
61
+ The base period of the RoPE embeddings.
62
+ rope_scaling (`Dict`, *optional*):
63
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
64
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
65
+ accordingly.
66
+ Expected contents:
67
+ `rope_type` (`str`):
68
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
69
+ 'llama3'], with 'default' being the original RoPE implementation.
70
+ `factor` (`float`, *optional*):
71
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
72
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
73
+ original maximum pre-trained length.
74
+ `original_max_position_embeddings` (`int`, *optional*):
75
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
76
+ pretraining.
77
+ `attention_factor` (`float`, *optional*):
78
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
79
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
80
+ `factor` field to infer the suggested value.
81
+ `beta_fast` (`float`, *optional*):
82
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
83
+ ramp function. If unspecified, it defaults to 32.
84
+ `beta_slow` (`float`, *optional*):
85
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
86
+ ramp function. If unspecified, it defaults to 1.
87
+ `short_factor` (`List[float]`, *optional*):
88
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
89
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
90
+ size divided by the number of attention heads divided by 2
91
+ `long_factor` (`List[float]`, *optional*):
92
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
93
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
94
+ size divided by the number of attention heads divided by 2
95
+ `low_freq_factor` (`float`, *optional*):
96
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
97
+ `high_freq_factor` (`float`, *optional*):
98
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
99
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
100
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
101
+ attention_dropout (`float`, *optional*, defaults to 0.0):
102
+ The dropout ratio for the attention probabilities.
103
+ sliding_window (`int`, *optional*, defaults to 4096):
104
+ Size of the sliding window attention context.
105
+ sliding_window_pattern (`int`, *optional*, defaults to 4):
106
+ Pattern for the sliding window attention.
107
+ cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.
108
+
109
+ ```python
110
+ >>> from transformers import Cohere2Model, Cohere2Config
111
+
112
+ >>> # Initializing a Cohere Nextmodel configuration
113
+ >>> configuration = Cohere2Config()
114
+
115
+ >>> # Initializing a model from the Cohere2 configuration
116
+ >>> model = Cohere2Model(configuration) # doctest: +SKIP
117
+
118
+ >>> # Accessing the model configuration
119
+ >>> configuration = model.config # doctest: +SKIP
120
+ ```
121
+ """
122
+
123
+ model_type = "cohere2"
124
+ keys_to_ignore_at_inference = ["past_key_values"]
125
+
126
+ def __init__(
127
+ self,
128
+ vocab_size=256000,
129
+ hidden_size=8192,
130
+ intermediate_size=22528,
131
+ logit_scale=0.0625,
132
+ num_hidden_layers=40,
133
+ num_attention_heads=64,
134
+ num_key_value_heads=None,
135
+ hidden_act="silu",
136
+ max_position_embeddings=8192,
137
+ initializer_range=0.02,
138
+ layer_norm_eps=1e-5,
139
+ use_cache=True,
140
+ pad_token_id=0,
141
+ bos_token_id=5,
142
+ eos_token_id=255001,
143
+ tie_word_embeddings=True,
144
+ rope_theta=10000.0,
145
+ rope_scaling=None,
146
+ attention_bias=False,
147
+ attention_dropout=0.0,
148
+ sliding_window=4096,
149
+ sliding_window_pattern=4,
150
+ cache_implementation="hybrid",
151
+ **kwargs,
152
+ ):
153
+ self.vocab_size = vocab_size
154
+ self.max_position_embeddings = max_position_embeddings
155
+ self.hidden_size = hidden_size
156
+ self.logit_scale = logit_scale
157
+ self.intermediate_size = intermediate_size
158
+ self.num_hidden_layers = num_hidden_layers
159
+ self.num_attention_heads = num_attention_heads
160
+
161
+ # for backward compatibility
162
+ if num_key_value_heads is None:
163
+ num_key_value_heads = num_attention_heads
164
+
165
+ self.num_key_value_heads = num_key_value_heads
166
+ self.hidden_act = hidden_act
167
+ self.initializer_range = initializer_range
168
+ self.layer_norm_eps = layer_norm_eps
169
+ self.use_cache = use_cache
170
+ self.rope_theta = rope_theta
171
+ self.rope_scaling = rope_scaling
172
+ self.attention_bias = attention_bias
173
+ self.attention_dropout = attention_dropout
174
+ self.sliding_window = sliding_window
175
+ self.sliding_window_pattern = sliding_window_pattern
176
+ # Need to specify head_dim in the config so it can be used in the attention forward functions
177
+ self.head_dim = hidden_size // num_attention_heads
178
+ self.cache_implementation = cache_implementation
179
+
180
+ # Validate the correctness of rotary position embeddings parameters
181
+ rope_config_validation(self)
182
+
183
+ super().__init__(
184
+ pad_token_id=pad_token_id,
185
+ bos_token_id=bos_token_id,
186
+ eos_token_id=eos_token_id,
187
+ tie_word_embeddings=tie_word_embeddings,
188
+ **kwargs,
189
+ )
190
+
191
+
192
+__all__ = ["Cohere2Config"]