@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 10e6322283
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Model] Fix and clean commandr (#3671)

Roy committed Mar 28, 2024 at 08:20 UTC 10e6322283a9149c23eb76db50e6da972ce4b99e
1 file changed +4 -12
vllm/model_executor/models/commandr.py
+4 -12
@@ -26,7 +26,6 @@ import torch
26 import torch.utils.checkpoint
27 from torch import nn
28 from transformers import CohereConfig
29 -from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
@@ -46,8 +45,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
45 hf_model_weights_iterator)
46 from vllm.sequence import SamplerOutput
47
49 -KVCache = Tuple[torch.Tensor, torch.Tensor]
50 -
48
49 class LayerNorm(nn.Module):
50
@@ -70,9 +67,6 @@ class LayerNorm(nn.Module):
67 return hidden_states.to(input_dtype), residuals
68
69
73 -ALL_LAYERNORM_LAYERS.append(LayerNorm)
74 -
75 -
70 # Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
71 class CohereMLP(nn.Module):
72
@@ -137,7 +131,6 @@ class CohereAttention(nn.Module):
131 self.max_position_embeddings = config.max_position_embeddings
132 self.rope_theta = config.rope_theta
133 self.rope_scaling = getattr(config, "rope_scaling", None)
140 - self.is_causal = True
134 self.qkv_proj = QKVParallelLinear(
135 self.hidden_size,
136 self.head_dim,
@@ -171,7 +164,7 @@ class CohereAttention(nn.Module):
164 self,
165 positions: torch.Tensor,
166 hidden_states: torch.Tensor,
174 - kv_cache: KVCache,
167 + kv_cache: torch.Tensor,
168 attn_metadata: AttentionMetadata,
169 ) -> torch.Tensor:
170 qkv, _ = self.qkv_proj(hidden_states)
@@ -200,7 +193,7 @@ class CohereDecoderLayer(nn.Module):
193 self,
194 positions: torch.Tensor,
195 hidden_states: torch.Tensor,
203 - kv_cache: KVCache,
196 + kv_cache: torch.Tensor,
197 attn_metadata: AttentionMetadata,
198 residual: Optional[torch.Tensor],
199 ) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -242,7 +235,7 @@ class CohereModel(nn.Module):
235 self,
236 input_ids: torch.Tensor,
237 positions: torch.Tensor,
245 - kv_caches: List[KVCache],
238 + kv_caches: List[torch.Tensor],
239 attn_metadata: AttentionMetadata,
240 ) -> torch.Tensor:
241 hidden_states = self.embed_tokens(input_ids)
@@ -269,7 +262,6 @@ class CohereForCausalLM(nn.Module):
262 ) -> None:
263 super().__init__()
264 self.config = config
272 - self.unpadded_vocab_size = config.vocab_size
265 self.linear_method = linear_method
266 self.logits_processor = LogitsProcessor(config.vocab_size,
267 scale=config.logit_scale)
@@ -281,7 +273,7 @@ class CohereForCausalLM(nn.Module):
273 self,
274 input_ids: torch.Tensor,
275 positions: torch.Tensor,
284 - kv_caches: List[KVCache],
276 + kv_caches: List[torch.Tensor],
277 attn_metadata: AttentionMetadata,
278 ) -> torch.Tensor:
279 hidden_states = self.model(input_ids, positions, kv_caches,