A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] Bump up transformers to v4.39.0 & Remove StarCoder2Config (#3551)
Co-authored-by: Roy <jasonailu87@gmail.com> Co-authored-by: Roger Meier <r.meier@siemens.com>
Woosuk Kwon committed
Mar 21, 2024 at 07:58 UTC
c188ecb080501c5ccb34bbd6542978284c547122
6 files changed
+3
-76
requirements-rocm.txt
+1
-1
@@ -7,7 +7,7 @@ ray >= 2.9
7
sentencepiece # Required for LLaMA tokenizer.
8
numpy
9
tokenizers>=0.15.0
10
-transformers >= 4.38.0 # Required for Gemma.
10
+transformers >= 4.39.0 # Required for StarCoder2.
11
fastapi
12
uvicorn[standard]
13
pydantic >= 2.0 # Required for OpenAI server.
requirements.txt
+1
-1
@@ -5,7 +5,7 @@ ray >= 2.9
5
sentencepiece # Required for LLaMA tokenizer.
6
numpy
7
torch == 2.1.2
8
-transformers >= 4.38.0 # Required for Gemma.
8
+transformers >= 4.39.0 # Required for StarCoder2.
9
xformers == 0.0.23.post1 # Required for CUDA 12.1.
10
fastapi
11
uvicorn[standard]
vllm/model_executor/models/starcoder2.py
+1
-7
@@ -22,6 +22,7 @@ from typing import List, Optional, Tuple
22
23
import torch
24
from torch import nn
25
+from transformers import Starcoder2Config
26
27
from vllm.model_executor.input_metadata import InputMetadata
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -42,13 +43,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
43
hf_model_weights_iterator)
44
from vllm.sequence import SamplerOutput
45
45
-try:
46
- from transformers import Starcoder2Config
47
-except ImportError:
48
- # fallback to PretrainedConfig
49
- # NOTE: Please install transformers from source or use transformers>=4.39.0
50
- from transformers import PretrainedConfig as Starcoder2Config
51
-
46
KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48
vllm/transformers_utils/config.py
-10
@@ -9,7 +9,6 @@ _CONFIG_REGISTRY = {
9
"mpt": MPTConfig,
10
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
12
- "starcoder2": Starcoder2Config,
12
"jais": JAISConfig,
13
}
14
@@ -18,15 +17,6 @@ def get_config(model: str,
17
trust_remote_code: bool,
18
revision: Optional[str] = None,
19
code_revision: Optional[str] = None) -> PretrainedConfig:
21
- # FIXME(woosuk): This is a temporary fix for StarCoder2.
22
- # Remove this when the model is supported by HuggingFace transformers.
23
- if "bigcode" in model and "starcoder2" in model:
24
- config_class = _CONFIG_REGISTRY["starcoder2"]
25
- config = config_class.from_pretrained(model,
26
- revision=revision,
27
- code_revision=code_revision)
28
- return config
29
-
20
try:
21
config = AutoConfig.from_pretrained(
22
model,
vllm/transformers_utils/configs/__init__.py
-2
@@ -4,13 +4,11 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
4
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
7
-from vllm.transformers_utils.configs.starcoder2 import Starcoder2Config
7
from vllm.transformers_utils.configs.jais import JAISConfig
8
9
__all__ = [
10
"ChatGLMConfig",
11
"MPTConfig",
12
"RWConfig",
14
- "Starcoder2Config",
13
"JAISConfig",
14
]
vllm/transformers_utils/configs/starcoder2.py
deleted
-55
@@ -1,55 +0,0 @@
1
-from transformers import PretrainedConfig
2
-
3
-
4
-class Starcoder2Config(PretrainedConfig):
5
- model_type = "starcoder2"
6
- keys_to_ignore_at_inference = ["past_key_values"]
7
-
8
- def __init__(
9
- self,
10
- vocab_size=49152,
11
- hidden_size=3072,
12
- intermediate_size=12288,
13
- num_hidden_layers=30,
14
- num_attention_heads=24,
15
- num_key_value_heads=2,
16
- hidden_act="gelu_pytorch_tanh",
17
- max_position_embeddings=4096,
18
- initializer_range=0.018042,
19
- norm_epsilon=1e-5,
20
- use_cache=True,
21
- bos_token_id=50256,
22
- eos_token_id=50256,
23
- rope_theta=10000.0,
24
- sliding_window=None,
25
- attention_dropout=0.0,
26
- residual_dropout=0.0,
27
- embedding_dropout=0.0,
28
- use_bias=True,
29
- **kwargs,
30
- ):
31
- self.vocab_size = vocab_size
32
- self.max_position_embeddings = max_position_embeddings
33
- self.hidden_size = hidden_size
34
- self.intermediate_size = intermediate_size
35
- self.num_hidden_layers = num_hidden_layers
36
- self.num_attention_heads = num_attention_heads
37
- self.sliding_window = sliding_window
38
- self.use_bias = use_bias
39
- self.num_key_value_heads = num_key_value_heads
40
- self.hidden_act = hidden_act
41
- self.initializer_range = initializer_range
42
- self.norm_epsilon = norm_epsilon
43
- self.use_cache = use_cache
44
- self.rope_theta = rope_theta
45
- self.attention_dropout = attention_dropout
46
- self.residual_dropout = residual_dropout
47
- self.embedding_dropout = embedding_dropout
48
-
49
- super().__init__(
50
- bos_token_id=bos_token_id,
51
- eos_token_id=eos_token_id,
52
- **kwargs,
53
- )
54
- if self.architectures is None:
55
- self.architectures = ['Starcoder2ForCausalLM']