A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Bump up transformers version & Remove MistralConfig (#1254)
Woosuk Kwon committed
Oct 13, 2023 at 10:05 UTC
e7c8555d0652e5213353411dd40b99408b1b1b28
6 files changed
+4
-81
requirements.txt
+2
-2
@@ -6,8 +6,8 @@ pyarrow # Required for Ray data.
6
sentencepiece # Required for LLaMA tokenizer.
7
numpy
8
torch == 2.0.1
9
-transformers >= 4.33.1 # Required for Code Llama.
10
-xformers == 0.0.22
9
+transformers >= 4.34.0 # Required for Mistral.
10
+xformers == 0.0.22 # Required for Mistral.
11
fastapi
12
uvicorn[standard]
13
pydantic < 2 # Required for OpenAI server.
vllm/model_executor/models/__init__.py
+1
-1
@@ -9,10 +9,10 @@ from vllm.model_executor.models.gpt_j import GPTJForCausalLM
9
from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10
from vllm.model_executor.models.internlm import InternLMForCausalLM
11
from vllm.model_executor.models.llama import LlamaForCausalLM
12
+from vllm.model_executor.models.mistral import MistralForCausalLM
13
from vllm.model_executor.models.mpt import MPTForCausalLM
14
from vllm.model_executor.models.opt import OPTForCausalLM
15
from vllm.model_executor.models.qwen import QWenLMHeadModel
15
-from vllm.model_executor.models.mistral import MistralForCausalLM
16
17
__all__ = [
18
"AquilaForCausalLM",
vllm/model_executor/models/mistral.py
+1
-1
@@ -29,6 +29,7 @@ from typing import List, Optional, Tuple
29
30
import torch
31
from torch import nn
32
+from transformers import MistralConfig
33
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -44,7 +45,6 @@ from vllm.model_executor.weight_utils import (
45
convert_pyslice_to_tensor, hf_model_weights_iterator,
46
load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47
from vllm.sequence import SamplerOutput
47
-from vllm.transformers_utils.configs.mistral import MistralConfig
48
49
KVCache = Tuple[torch.Tensor, torch.Tensor]
50
vllm/transformers_utils/config.py
-9
@@ -17,15 +17,6 @@ _CONFIG_REGISTRY = {
17
def get_config(model: str,
18
trust_remote_code: bool,
19
revision: Optional[str] = None) -> PretrainedConfig:
20
- # NOTE: Because the Mistral model in HF hub does not have
21
- # `configuration_mistral.py`, we cannot use `AutoConfig` to load the
22
- # config. Instead, we use `MistralConfig` directly.
23
- # NOTE: This is a hack. This does not work for local models.
24
- # FIXME: Remove this once the Mistral model is available in the stable
25
- # version of HF transformers.
26
- if "mistral" in model.lower():
27
- return MistralConfig.from_pretrained(model, revision=revision)
28
-
20
try:
21
config = AutoConfig.from_pretrained(
22
model, trust_remote_code=trust_remote_code, revision=revision)
vllm/transformers_utils/configs/__init__.py
-2
@@ -6,7 +6,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7
# `FalconConfig` class from the official HuggingFace transformers library.
8
from vllm.transformers_utils.configs.falcon import RWConfig
9
-from vllm.transformers_utils.configs.mistral import MistralConfig
9
10
__all__ = [
11
"MPTConfig",
@@ -14,5 +13,4 @@ __all__ = [
13
"AquilaConfig",
14
"QWenConfig",
15
"RWConfig",
17
- "MistralConfig",
16
]
vllm/transformers_utils/configs/mistral.py
deleted
-66
@@ -1,66 +0,0 @@
1
-# Licensed under the Apache License, Version 2.0 (the "License");
2
-# you may not use this file except in compliance with the License.
3
-# You may obtain a copy of the License at
4
-#
5
-# http://www.apache.org/licenses/LICENSE-2.0
6
-#
7
-# Unless required by applicable law or agreed to in writing, software
8
-# distributed under the License is distributed on an "AS IS" BASIS,
9
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
-# See the License for the specific language governing permissions and
11
-# limitations under the License.
12
-"""Mistral-7B-v0.1 configuration"""
13
-from transformers.configuration_utils import PretrainedConfig
14
-
15
-
16
-class MistralConfig(PretrainedConfig):
17
- model_type = "mistral"
18
- keys_to_ignore_at_inference = ["past_key_values"]
19
-
20
- def __init__(
21
- self,
22
- vocab_size=32000,
23
- hidden_size=4096,
24
- intermediate_size=14336,
25
- num_hidden_layers=32,
26
- num_attention_heads=32,
27
- num_key_value_heads=8,
28
- hidden_act="silu",
29
- max_position_embeddings=4096 * 32,
30
- initializer_range=0.02,
31
- rms_norm_eps=1e-6,
32
- use_cache=True,
33
- pad_token_id=None,
34
- bos_token_id=1,
35
- eos_token_id=2,
36
- tie_word_embeddings=False,
37
- rope_theta=10000.0,
38
- sliding_window=4096,
39
- **kwargs,
40
- ):
41
- self.vocab_size = vocab_size
42
- self.max_position_embeddings = max_position_embeddings
43
- self.hidden_size = hidden_size
44
- self.intermediate_size = intermediate_size
45
- self.num_hidden_layers = num_hidden_layers
46
- self.num_attention_heads = num_attention_heads
47
- self.sliding_window = sliding_window
48
-
49
- # for backward compatibility
50
- if num_key_value_heads is None:
51
- num_key_value_heads = num_attention_heads
52
-
53
- self.num_key_value_heads = num_key_value_heads
54
- self.hidden_act = hidden_act
55
- self.initializer_range = initializer_range
56
- self.rms_norm_eps = rms_norm_eps
57
- self.use_cache = use_cache
58
- self.rope_theta = rope_theta
59
-
60
- super().__init__(
61
- pad_token_id=pad_token_id,
62
- bos_token_id=bos_token_id,
63
- eos_token_id=eos_token_id,
64
- tie_word_embeddings=tie_word_embeddings,
65
- **kwargs,
66
- )