A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Doc] Add Nemotron to supported model docs (#6843)
Michael Goin committed
Jul 26, 2024 at 17:32 UTC
281977bd6eccade50be461f5a22cc51b74006976
2 files changed
+5
-3
docs/source/models/supported_models.rst
+4
@@ -113,6 +113,10 @@ Decoder-only Language Models
113
- MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
114
- :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
115
-
116
+ * - :code:`NemotronForCausalLM`
117
+ - Nemotron-3, Nemotron-4, Minitron
118
+ - :code:`nvidia/Minitron-8B-Base`, :code:`mgoin/Nemotron-4-340B-Base-hf-FP8`, etc.
119
+ - ✅︎
120
* - :code:`OLMoForCausalLM`
121
- OLMo
122
- :code:`allenai/OLMo-1B-hf`, :code:`allenai/OLMo-7B-hf`, etc.
vllm/model_executor/layers/activation.py
+1
-3
@@ -166,9 +166,7 @@ class ReLUSquaredActivation(CustomOp):
166
167
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
168
"""PyTorch-native implementation equivalent to forward()."""
169
- relu_applied = nn.functional.relu(x)
170
- squared = torch.square(relu_applied)
171
- return squared
169
+ return torch.square(F.relu(x))
170
171
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
172
return self.forward_native(x)