@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c188ecb080
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] Bump up transformers to v4.39.0 & Remove StarCoder2Config (#3551)

Co-authored-by: Roy <jasonailu87@gmail.com> Co-authored-by: Roger Meier <r.meier@siemens.com>

Woosuk Kwon committed Mar 21, 2024 at 07:58 UTC c188ecb080501c5ccb34bbd6542978284c547122
6 files changed +3 -76
requirements-rocm.txt
+1 -1
@@ -7,7 +7,7 @@ ray >= 2.9
7 sentencepiece # Required for LLaMA tokenizer.
8 numpy
9 tokenizers>=0.15.0
10 -transformers >= 4.38.0 # Required for Gemma.
10 +transformers >= 4.39.0 # Required for StarCoder2.
11 fastapi
12 uvicorn[standard]
13 pydantic >= 2.0 # Required for OpenAI server.
requirements.txt
+1 -1
@@ -5,7 +5,7 @@ ray >= 2.9
5 sentencepiece # Required for LLaMA tokenizer.
6 numpy
7 torch == 2.1.2
8 -transformers >= 4.38.0 # Required for Gemma.
8 +transformers >= 4.39.0 # Required for StarCoder2.
9 xformers == 0.0.23.post1 # Required for CUDA 12.1.
10 fastapi
11 uvicorn[standard]
vllm/model_executor/models/starcoder2.py
+1 -7
@@ -22,6 +22,7 @@ from typing import List, Optional, Tuple
22
23 import torch
24 from torch import nn
25 +from transformers import Starcoder2Config
26
27 from vllm.model_executor.input_metadata import InputMetadata
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -42,13 +43,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
43 hf_model_weights_iterator)
44 from vllm.sequence import SamplerOutput
45
45 -try:
46 - from transformers import Starcoder2Config
47 -except ImportError:
48 - # fallback to PretrainedConfig
49 - # NOTE: Please install transformers from source or use transformers>=4.39.0
50 - from transformers import PretrainedConfig as Starcoder2Config
51 -
46 KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48
vllm/transformers_utils/config.py
-10
@@ -9,7 +9,6 @@ _CONFIG_REGISTRY = {
9 "mpt": MPTConfig,
10 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
12 - "starcoder2": Starcoder2Config,
12 "jais": JAISConfig,
13 }
14
@@ -18,15 +17,6 @@ def get_config(model: str,
17 trust_remote_code: bool,
18 revision: Optional[str] = None,
19 code_revision: Optional[str] = None) -> PretrainedConfig:
21 - # FIXME(woosuk): This is a temporary fix for StarCoder2.
22 - # Remove this when the model is supported by HuggingFace transformers.
23 - if "bigcode" in model and "starcoder2" in model:
24 - config_class = _CONFIG_REGISTRY["starcoder2"]
25 - config = config_class.from_pretrained(model,
26 - revision=revision,
27 - code_revision=code_revision)
28 - return config
29 -
20 try:
21 config = AutoConfig.from_pretrained(
22 model,
vllm/transformers_utils/configs/__init__.py
-2
@@ -4,13 +4,11 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
4 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 -from vllm.transformers_utils.configs.starcoder2 import Starcoder2Config
7 from vllm.transformers_utils.configs.jais import JAISConfig
8
9 __all__ = [
10 "ChatGLMConfig",
11 "MPTConfig",
12 "RWConfig",
14 - "Starcoder2Config",
13 "JAISConfig",
14 ]
vllm/transformers_utils/configs/starcoder2.py deleted
-55
@@ -1,55 +0,0 @@
1 -from transformers import PretrainedConfig
2 -
3 -
4 -class Starcoder2Config(PretrainedConfig):
5 - model_type = "starcoder2"
6 - keys_to_ignore_at_inference = ["past_key_values"]
7 -
8 - def __init__(
9 - self,
10 - vocab_size=49152,
11 - hidden_size=3072,
12 - intermediate_size=12288,
13 - num_hidden_layers=30,
14 - num_attention_heads=24,
15 - num_key_value_heads=2,
16 - hidden_act="gelu_pytorch_tanh",
17 - max_position_embeddings=4096,
18 - initializer_range=0.018042,
19 - norm_epsilon=1e-5,
20 - use_cache=True,
21 - bos_token_id=50256,
22 - eos_token_id=50256,
23 - rope_theta=10000.0,
24 - sliding_window=None,
25 - attention_dropout=0.0,
26 - residual_dropout=0.0,
27 - embedding_dropout=0.0,
28 - use_bias=True,
29 - **kwargs,
30 - ):
31 - self.vocab_size = vocab_size
32 - self.max_position_embeddings = max_position_embeddings
33 - self.hidden_size = hidden_size
34 - self.intermediate_size = intermediate_size
35 - self.num_hidden_layers = num_hidden_layers
36 - self.num_attention_heads = num_attention_heads
37 - self.sliding_window = sliding_window
38 - self.use_bias = use_bias
39 - self.num_key_value_heads = num_key_value_heads
40 - self.hidden_act = hidden_act
41 - self.initializer_range = initializer_range
42 - self.norm_epsilon = norm_epsilon
43 - self.use_cache = use_cache
44 - self.rope_theta = rope_theta
45 - self.attention_dropout = attention_dropout
46 - self.residual_dropout = residual_dropout
47 - self.embedding_dropout = embedding_dropout
48 -
49 - super().__init__(
50 - bos_token_id=bos_token_id,
51 - eos_token_id=eos_token_id,
52 - **kwargs,
53 - )
54 - if self.architectures is None:
55 - self.architectures = ['Starcoder2ForCausalLM']