A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Minor fixes for Mixtral (#2015)
Woosuk Kwon committed
Dec 11, 2023 at 09:16 UTC
4ff0203987ff100eaaad69f0a8abf7ed821e3a0a
2 files changed
+5
-6
docs/source/models/supported_models.rst
+3
@@ -50,6 +50,9 @@ Alongside each architecture, we include some popular models that use it.
50
* - :code:`MistralForCausalLM`
51
- Mistral, Mistral-Instruct
52
- :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
53
+ * - :code:`MixtralForCausalLM`
54
+ - Mixtral-8x7B, Mixtral-8x7B-Instruct
55
+ - :code:`mistralai/Mixtral-8x7B-v0.1`, :code:`mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.
56
* - :code:`MPTForCausalLM`
57
- MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
58
- :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
vllm/model_executor/models/mixtral.py
+2
-6
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Mixtral model."""
24
-from typing import List, Optional, Tuple, Union
24
+from typing import List, Optional, Tuple
25
26
import numpy as np
27
@@ -453,10 +453,6 @@ class MixtralForCausalLM(nn.Module):
453
assert linear_method is None
454
self.padding_idx = config.pad_token_id
455
self.vocab_size = config.vocab_size
456
- self.tok_embeddings: Union[nn.Embedding, None] = None
457
- self.layers: nn.ModuleList = None
458
- self.output: Union[nn.Linear, None] = None
459
- self.sampler: Union[Sampler, None] = None
456
self.tok_embeddings = VocabParallelEmbedding(
457
config.vocab_size,
458
config.hidden_size,
@@ -492,6 +488,7 @@ class MixtralForCausalLM(nn.Module):
488
input_metadata,
489
cache_event,
490
)
491
+ hidden_states = self.norm(hidden_states)
492
return hidden_states
493
494
def sample(
@@ -499,7 +496,6 @@ class MixtralForCausalLM(nn.Module):
496
hidden_states: Optional[torch.Tensor],
497
sampling_metadata: SamplingMetadata,
498
) -> SamplerOutput:
502
- hidden_states = self.norm(hidden_states)
499
next_tokens = self.sampler(self.output.weight, hidden_states,
500
sampling_metadata)
501
return next_tokens