A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Model] Fix and clean commandr (#3671)
Roy committed
Mar 28, 2024 at 08:20 UTC
10e6322283a9149c23eb76db50e6da972ce4b99e
1 file changed
+4
-12
vllm/model_executor/models/commandr.py
+4
-12
@@ -26,7 +26,6 @@ import torch
26
import torch.utils.checkpoint
27
from torch import nn
28
from transformers import CohereConfig
29
-from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
@@ -46,8 +45,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
45
hf_model_weights_iterator)
46
from vllm.sequence import SamplerOutput
47
49
-KVCache = Tuple[torch.Tensor, torch.Tensor]
50
-
48
49
class LayerNorm(nn.Module):
50
@@ -70,9 +67,6 @@ class LayerNorm(nn.Module):
67
return hidden_states.to(input_dtype), residuals
68
69
73
-ALL_LAYERNORM_LAYERS.append(LayerNorm)
74
-
75
-
70
# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
71
class CohereMLP(nn.Module):
72
@@ -137,7 +131,6 @@ class CohereAttention(nn.Module):
131
self.max_position_embeddings = config.max_position_embeddings
132
self.rope_theta = config.rope_theta
133
self.rope_scaling = getattr(config, "rope_scaling", None)
140
- self.is_causal = True
134
self.qkv_proj = QKVParallelLinear(
135
self.hidden_size,
136
self.head_dim,
@@ -171,7 +164,7 @@ class CohereAttention(nn.Module):
164
self,
165
positions: torch.Tensor,
166
hidden_states: torch.Tensor,
174
- kv_cache: KVCache,
167
+ kv_cache: torch.Tensor,
168
attn_metadata: AttentionMetadata,
169
) -> torch.Tensor:
170
qkv, _ = self.qkv_proj(hidden_states)
@@ -200,7 +193,7 @@ class CohereDecoderLayer(nn.Module):
193
self,
194
positions: torch.Tensor,
195
hidden_states: torch.Tensor,
203
- kv_cache: KVCache,
196
+ kv_cache: torch.Tensor,
197
attn_metadata: AttentionMetadata,
198
residual: Optional[torch.Tensor],
199
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -242,7 +235,7 @@ class CohereModel(nn.Module):
235
self,
236
input_ids: torch.Tensor,
237
positions: torch.Tensor,
245
- kv_caches: List[KVCache],
238
+ kv_caches: List[torch.Tensor],
239
attn_metadata: AttentionMetadata,
240
) -> torch.Tensor:
241
hidden_states = self.embed_tokens(input_ids)
@@ -269,7 +262,6 @@ class CohereForCausalLM(nn.Module):
262
) -> None:
263
super().__init__()
264
self.config = config
272
- self.unpadded_vocab_size = config.vocab_size
265
self.linear_method = linear_method
266
self.logits_processor = LogitsProcessor(config.vocab_size,
267
scale=config.logit_scale)
@@ -281,7 +273,7 @@ class CohereForCausalLM(nn.Module):
273
self,
274
input_ids: torch.Tensor,
275
positions: torch.Tensor,
284
- kv_caches: List[KVCache],
276
+ kv_caches: List[torch.Tensor],
277
attn_metadata: AttentionMetadata,
278
) -> torch.Tensor:
279
hidden_states = self.model(input_ids, positions, kv_caches,