@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bddbbcb132
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] Support Cohere2ForCausalLM (Cohere R7B) (#11203)

Jani Monoses committed Dec 16, 2024 at 11:56 UTC bddbbcb132429084ede62855bcd6a1023a3645c1
5 files changed +26 -4
docs/source/models/supported_models.rst
+2 -2
@@ -118,9 +118,9 @@ Text Generation (``--task generate``)
118 - :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
119 - ✅︎
120 - ✅︎
121 - * - :code:`CohereForCausalLM`
121 + * - :code:`CohereForCausalLM`,:code:`Cohere2ForCausalLM`
122 - Command-R
123 - - :code:`CohereForAI/c4ai-command-r-v01`, etc.
123 + - :code:`CohereForAI/c4ai-command-r-v01`, :code:`CohereForAI/c4ai-command-r7b-12-2024`, etc.
124 - ✅︎
125 - ✅︎
126 * - :code:`DbrxForCausalLM`
tests/models/registry.py
+2
@@ -53,6 +53,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
53 # ChatGLMModel supports multimodal
54 "CohereForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r-v01",
55 trust_remote_code=True),
56 + "Cohere2ForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r7b-12-2024", # noqa: E501
57 + trust_remote_code=True),
58 "DbrxForCausalLM": _HfExamplesInfo("databricks/dbrx-instruct"),
59 "DeciLMForCausalLM": _HfExamplesInfo("Deci/DeciLM-7B-instruct",
60 trust_remote_code=True),
tests/models/test_initialization.py
+4
@@ -1,6 +1,7 @@
1 from unittest.mock import patch
2
3 import pytest
4 +import transformers
5 from transformers import PretrainedConfig
6
7 from vllm import LLM
@@ -11,6 +12,9 @@ from .registry import HF_EXAMPLE_MODELS
12 @pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
13 def test_can_initialize(model_arch):
14 model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
15 + if (model_arch == "Cohere2ForCausalLM"
16 + and transformers.__version__ < "4.48.0"):
17 + pytest.skip(reason="Model introduced in HF >= 4.48.0")
18 if not model_info.is_available_online:
19 pytest.skip("Model is not available online")
20
vllm/model_executor/models/commandr.py
+17 -2
@@ -48,7 +48,7 @@ from vllm.model_executor.utils import set_weight_attrs
48 from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
51 -from .utils import (is_pp_missing_parameter,
51 +from .utils import (extract_layer_index, is_pp_missing_parameter,
52 make_empty_intermediate_tensors_factory, make_layers,
53 maybe_prefix)
54
@@ -171,12 +171,26 @@ class CohereAttention(nn.Module):
171 rope_scaling=self.rope_scaling,
172 is_neox_style=False,
173 )
174 +
175 + sliding_window = getattr(config, "sliding_window", None)
176 + # Model v2 has sliding windows, v1 does not
177 + self.v1 = sliding_window is None
178 +
179 + layer_idx = extract_layer_index(prefix)
180 + layer_has_sliding_window = (
181 + getattr(config, "sliding_window_pattern", False)
182 + and (layer_idx + 1) % self.config.sliding_window_pattern != 0)
183 +
184 + self.sliding_window = (sliding_window
185 + if layer_has_sliding_window else None)
186 +
187 self.attn = Attention(self.num_heads,
188 self.head_dim,
189 self.scaling,
190 num_kv_heads=self.num_kv_heads,
191 cache_config=cache_config,
192 quant_config=quant_config,
193 + per_layer_sliding_window=self.sliding_window,
194 prefix=f"{prefix}.attn")
195 if self.use_qk_norm:
196 self.q_norm = LayerNorm(param_shape=(self.num_heads,
@@ -206,7 +220,8 @@ class CohereAttention(nn.Module):
220 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
221 if self.use_qk_norm:
222 q, k = self._apply_qk_norm(q, k)
209 - q, k = self.rotary_emb(positions, q, k)
223 + if self.v1 or self.sliding_window:
224 + q, k = self.rotary_emb(positions, q, k)
225 attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
226 output, _ = self.o_proj(attn_output)
227 return output
vllm/model_executor/models/registry.py
+1
@@ -41,6 +41,7 @@ _TEXT_GENERATION_MODELS = {
41 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
42 # ChatGLMModel supports multimodal
43 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
44 + "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
45 "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
46 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
47 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),