@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7c8555d06
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Bump up transformers version & Remove MistralConfig (#1254)

Woosuk Kwon committed Oct 13, 2023 at 10:05 UTC e7c8555d0652e5213353411dd40b99408b1b1b28
6 files changed +4 -81
requirements.txt
+2 -2
@@ -6,8 +6,8 @@ pyarrow # Required for Ray data.
6 sentencepiece # Required for LLaMA tokenizer.
7 numpy
8 torch == 2.0.1
9 -transformers >= 4.33.1 # Required for Code Llama.
10 -xformers == 0.0.22
9 +transformers >= 4.34.0 # Required for Mistral.
10 +xformers == 0.0.22 # Required for Mistral.
11 fastapi
12 uvicorn[standard]
13 pydantic < 2 # Required for OpenAI server.
vllm/model_executor/models/__init__.py
+1 -1
@@ -9,10 +9,10 @@ from vllm.model_executor.models.gpt_j import GPTJForCausalLM
9 from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10 from vllm.model_executor.models.internlm import InternLMForCausalLM
11 from vllm.model_executor.models.llama import LlamaForCausalLM
12 +from vllm.model_executor.models.mistral import MistralForCausalLM
13 from vllm.model_executor.models.mpt import MPTForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 from vllm.model_executor.models.qwen import QWenLMHeadModel
15 -from vllm.model_executor.models.mistral import MistralForCausalLM
16
17 __all__ = [
18 "AquilaForCausalLM",
vllm/model_executor/models/mistral.py
+1 -1
@@ -29,6 +29,7 @@ from typing import List, Optional, Tuple
29
30 import torch
31 from torch import nn
32 +from transformers import MistralConfig
33
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -44,7 +45,6 @@ from vllm.model_executor.weight_utils import (
45 convert_pyslice_to_tensor, hf_model_weights_iterator,
46 load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47 from vllm.sequence import SamplerOutput
47 -from vllm.transformers_utils.configs.mistral import MistralConfig
48
49 KVCache = Tuple[torch.Tensor, torch.Tensor]
50
vllm/transformers_utils/config.py
-9
@@ -17,15 +17,6 @@ _CONFIG_REGISTRY = {
17 def get_config(model: str,
18 trust_remote_code: bool,
19 revision: Optional[str] = None) -> PretrainedConfig:
20 - # NOTE: Because the Mistral model in HF hub does not have
21 - # `configuration_mistral.py`, we cannot use `AutoConfig` to load the
22 - # config. Instead, we use `MistralConfig` directly.
23 - # NOTE: This is a hack. This does not work for local models.
24 - # FIXME: Remove this once the Mistral model is available in the stable
25 - # version of HF transformers.
26 - if "mistral" in model.lower():
27 - return MistralConfig.from_pretrained(model, revision=revision)
28 -
20 try:
21 config = AutoConfig.from_pretrained(
22 model, trust_remote_code=trust_remote_code, revision=revision)
vllm/transformers_utils/configs/__init__.py
-2
@@ -6,7 +6,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7 # `FalconConfig` class from the official HuggingFace transformers library.
8 from vllm.transformers_utils.configs.falcon import RWConfig
9 -from vllm.transformers_utils.configs.mistral import MistralConfig
9
10 __all__ = [
11 "MPTConfig",
@@ -14,5 +13,4 @@ __all__ = [
13 "AquilaConfig",
14 "QWenConfig",
15 "RWConfig",
17 - "MistralConfig",
16 ]
vllm/transformers_utils/configs/mistral.py deleted
-66
@@ -1,66 +0,0 @@
1 -# Licensed under the Apache License, Version 2.0 (the "License");
2 -# you may not use this file except in compliance with the License.
3 -# You may obtain a copy of the License at
4 -#
5 -# http://www.apache.org/licenses/LICENSE-2.0
6 -#
7 -# Unless required by applicable law or agreed to in writing, software
8 -# distributed under the License is distributed on an "AS IS" BASIS,
9 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 -# See the License for the specific language governing permissions and
11 -# limitations under the License.
12 -"""Mistral-7B-v0.1 configuration"""
13 -from transformers.configuration_utils import PretrainedConfig
14 -
15 -
16 -class MistralConfig(PretrainedConfig):
17 - model_type = "mistral"
18 - keys_to_ignore_at_inference = ["past_key_values"]
19 -
20 - def __init__(
21 - self,
22 - vocab_size=32000,
23 - hidden_size=4096,
24 - intermediate_size=14336,
25 - num_hidden_layers=32,
26 - num_attention_heads=32,
27 - num_key_value_heads=8,
28 - hidden_act="silu",
29 - max_position_embeddings=4096 * 32,
30 - initializer_range=0.02,
31 - rms_norm_eps=1e-6,
32 - use_cache=True,
33 - pad_token_id=None,
34 - bos_token_id=1,
35 - eos_token_id=2,
36 - tie_word_embeddings=False,
37 - rope_theta=10000.0,
38 - sliding_window=4096,
39 - **kwargs,
40 - ):
41 - self.vocab_size = vocab_size
42 - self.max_position_embeddings = max_position_embeddings
43 - self.hidden_size = hidden_size
44 - self.intermediate_size = intermediate_size
45 - self.num_hidden_layers = num_hidden_layers
46 - self.num_attention_heads = num_attention_heads
47 - self.sliding_window = sliding_window
48 -
49 - # for backward compatibility
50 - if num_key_value_heads is None:
51 - num_key_value_heads = num_attention_heads
52 -
53 - self.num_key_value_heads = num_key_value_heads
54 - self.hidden_act = hidden_act
55 - self.initializer_range = initializer_range
56 - self.rms_norm_eps = rms_norm_eps
57 - self.use_cache = use_cache
58 - self.rope_theta = rope_theta
59 -
60 - super().__init__(
61 - pad_token_id=pad_token_id,
62 - bos_token_id=bos_token_id,
63 - eos_token_id=eos_token_id,
64 - tie_word_embeddings=tie_word_embeddings,
65 - **kwargs,
66 - )