@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ff0203987
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Minor fixes for Mixtral (#2015)

Woosuk Kwon committed Dec 11, 2023 at 09:16 UTC 4ff0203987ff100eaaad69f0a8abf7ed821e3a0a
2 files changed +5 -6
docs/source/models/supported_models.rst
+3
@@ -50,6 +50,9 @@ Alongside each architecture, we include some popular models that use it.
50 * - :code:`MistralForCausalLM`
51 - Mistral, Mistral-Instruct
52 - :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
53 + * - :code:`MixtralForCausalLM`
54 + - Mixtral-8x7B, Mixtral-8x7B-Instruct
55 + - :code:`mistralai/Mixtral-8x7B-v0.1`, :code:`mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.
56 * - :code:`MPTForCausalLM`
57 - MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
58 - :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
vllm/model_executor/models/mixtral.py
+2 -6
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Mixtral model."""
24 -from typing import List, Optional, Tuple, Union
24 +from typing import List, Optional, Tuple
25
26 import numpy as np
27
@@ -453,10 +453,6 @@ class MixtralForCausalLM(nn.Module):
453 assert linear_method is None
454 self.padding_idx = config.pad_token_id
455 self.vocab_size = config.vocab_size
456 - self.tok_embeddings: Union[nn.Embedding, None] = None
457 - self.layers: nn.ModuleList = None
458 - self.output: Union[nn.Linear, None] = None
459 - self.sampler: Union[Sampler, None] = None
456 self.tok_embeddings = VocabParallelEmbedding(
457 config.vocab_size,
458 config.hidden_size,
@@ -492,6 +488,7 @@ class MixtralForCausalLM(nn.Module):
488 input_metadata,
489 cache_event,
490 )
491 + hidden_states = self.norm(hidden_states)
492 return hidden_states
493
494 def sample(
@@ -499,7 +496,6 @@ class MixtralForCausalLM(nn.Module):
496 hidden_states: Optional[torch.Tensor],
497 sampling_metadata: SamplingMetadata,
498 ) -> SamplerOutput:
502 - hidden_states = self.norm(hidden_states)
499 next_tokens = self.sampler(self.output.weight, hidden_states,
500 sampling_metadata)
501 return next_tokens