A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
Bump `transformers` version for Llama 3.1 hotfix and patch Chameleon (#6690)
Roger Wang committed
Jul 23, 2024 at 13:47 UTC
1bedf210e35f5a11df5e9dd51e82b0663f854cf4
7 files changed
+33
-178
requirements-common.txt
+1
-1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6
requests
7
tqdm
8
py-cpuinfo
9
-transformers >= 4.42.4 # Required for Gemma 2 and for additional chat template parameters.
9
+transformers >= 4.43.1 # Required for Chameleon and Llama 3.1 hotfox.
10
tokenizers >= 0.19.1 # Required for Llama 3.
11
fastapi
12
aiohttp
tests/test_config.py
+27
-26
@@ -64,9 +64,8 @@ def test_get_sliding_window():
64
65
66
def test_rope_customization():
67
- TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
67
+ TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
68
TEST_ROPE_THETA = 16_000_000.0
69
- LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
69
70
llama_model_config = ModelConfig(
71
"meta-llama/Meta-Llama-3-8B-Instruct",
@@ -96,27 +95,29 @@ def test_rope_customization():
95
None) == TEST_ROPE_THETA
96
assert llama_model_config.max_model_len == 16384
97
99
- longchat_model_config = ModelConfig(
100
- "lmsys/longchat-13b-16k",
101
- "lmsys/longchat-13b-16k",
102
- tokenizer_mode="auto",
103
- trust_remote_code=False,
104
- dtype="float16",
105
- seed=0,
106
- )
107
- assert getattr(longchat_model_config.hf_config, "rope_scaling",
108
- None) == LONGCHAT_ROPE_SCALING
109
- assert longchat_model_config.max_model_len == 16384
110
-
111
- longchat_model_config = ModelConfig(
112
- "lmsys/longchat-13b-16k",
113
- "lmsys/longchat-13b-16k",
114
- tokenizer_mode="auto",
115
- trust_remote_code=False,
116
- dtype="float16",
117
- seed=0,
118
- rope_scaling=TEST_ROPE_SCALING,
119
- )
120
- assert getattr(longchat_model_config.hf_config, "rope_scaling",
121
- None) == TEST_ROPE_SCALING
122
- assert longchat_model_config.max_model_len == 4096
98
+ # TODO: add these back when the rope configs are fixed
99
+ # LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
100
+ # longchat_model_config = ModelConfig(
101
+ # "lmsys/longchat-13b-16k",
102
+ # "lmsys/longchat-13b-16k",
103
+ # tokenizer_mode="auto",
104
+ # trust_remote_code=False,
105
+ # dtype="float16",
106
+ # seed=0,
107
+ # )
108
+ # assert getattr(longchat_model_config.hf_config, "rope_scaling",
109
+ # None) == LONGCHAT_ROPE_SCALING
110
+ # assert longchat_model_config.max_model_len == 16384
111
+
112
+ # longchat_model_config = ModelConfig(
113
+ # "lmsys/longchat-13b-16k",
114
+ # "lmsys/longchat-13b-16k",
115
+ # tokenizer_mode="auto",
116
+ # trust_remote_code=False,
117
+ # dtype="float16",
118
+ # seed=0,
119
+ # rope_scaling=TEST_ROPE_SCALING,
120
+ # )
121
+ # assert getattr(longchat_model_config.hf_config, "rope_scaling",
122
+ # None) == TEST_ROPE_SCALING
123
+ # assert longchat_model_config.max_model_len == 4096
vllm/model_executor/models/__init__.py
-2
@@ -16,8 +16,6 @@ _GENERATION_MODELS = {
16
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19
- #TODO(ywang96): remove this when huggingface fixes the model repo
20
- "ChameleonForCausalLM": ("chameleon", "ChameleonForConditionalGeneration"),
19
"ChameleonForConditionalGeneration":
20
("chameleon", "ChameleonForConditionalGeneration"),
21
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
vllm/model_executor/models/chameleon.py
+1
-2
@@ -6,6 +6,7 @@ import torch
6
import torch.nn.functional as F
7
from PIL import Image
8
from torch import nn
9
+from transformers import ChameleonConfig, ChameleonVQVAEConfig
10
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
@@ -30,8 +31,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
31
from vllm.multimodal.image import (cached_get_tokenizer,
32
repeat_and_pad_image_tokens)
33
from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
33
-from vllm.transformers_utils.configs import (ChameleonConfig,
34
- ChameleonVQVAEConfig)
34
from vllm.utils import print_warning_once
35
36
from .interfaces import SupportsVision
vllm/transformers_utils/config.py
+4
-5
@@ -5,10 +5,10 @@ from transformers import GenerationConfig, PretrainedConfig
5
6
from vllm.envs import VLLM_USE_MODELSCOPE
7
from vllm.logger import init_logger
8
-from vllm.transformers_utils.configs import (ChameleonConfig, ChatGLMConfig,
9
- DbrxConfig, JAISConfig,
10
- MedusaConfig, MLPSpeculatorConfig,
11
- MPTConfig, RWConfig)
8
+from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
+ JAISConfig, MedusaConfig,
10
+ MLPSpeculatorConfig, MPTConfig,
11
+ RWConfig)
12
13
if VLLM_USE_MODELSCOPE:
14
from modelscope import AutoConfig
@@ -18,7 +18,6 @@ else:
18
logger = init_logger(__name__)
19
20
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
21
- "chameleon": ChameleonConfig,
21
"chatglm": ChatGLMConfig,
22
"dbrx": DbrxConfig,
23
"mpt": MPTConfig,
vllm/transformers_utils/configs/__init__.py
-4
@@ -1,5 +1,3 @@
1
-from vllm.transformers_utils.configs.chameleon import (ChameleonConfig,
2
- ChameleonVQVAEConfig)
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.dbrx import DbrxConfig
3
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -12,8 +10,6 @@ from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
10
from vllm.transformers_utils.configs.mpt import MPTConfig
11
12
__all__ = [
15
- "ChameleonConfig",
16
- "ChameleonVQVAEConfig",
13
"ChatGLMConfig",
14
"DbrxConfig",
15
"MPTConfig",
vllm/transformers_utils/configs/chameleon.py
deleted
-138
@@ -1,138 +0,0 @@
1
-from typing import List, Optional
2
-
3
-from transformers import PretrainedConfig
4
-
5
-
6
-#TODO (ywang96): Remove this file and import it from
7
-# transformers once the new release with Chameleon support
8
-# is available.
9
-class ChameleonConfig(PretrainedConfig):
10
- model_type = "chameleon"
11
- keys_to_ignore_at_inference = ["past_key_values"]
12
-
13
- def __init__(
14
- self,
15
- vocab_size=65536,
16
- hidden_size=4096,
17
- intermediate_size=11008,
18
- num_hidden_layers=32,
19
- num_attention_heads=32,
20
- num_key_value_heads=32,
21
- hidden_act="silu",
22
- max_position_embeddings=4096,
23
- initializer_range=0.02,
24
- rms_norm_eps=1e-05,
25
- use_cache=True,
26
- pad_token_id=None,
27
- bos_token_id=1,
28
- eos_token_id=2,
29
- tie_word_embeddings=False,
30
- rope_theta=10000.0,
31
- rope_scaling=None,
32
- attention_bias=False,
33
- attention_dropout=0.0,
34
- model_parallel_size=1,
35
- swin_norm=False,
36
- vq_config=None,
37
- vocabulary_map=None,
38
- mlp_bias=False,
39
- **kwargs,
40
- ):
41
- self.vocab_size = vocab_size
42
- self.max_position_embeddings = max_position_embeddings
43
- self.hidden_size = hidden_size
44
- self.intermediate_size = intermediate_size
45
- self.num_hidden_layers = num_hidden_layers
46
- self.num_attention_heads = num_attention_heads
47
- self.mlp_bias = mlp_bias
48
-
49
- self.num_key_value_heads = num_key_value_heads
50
- self.hidden_act = hidden_act
51
- self.initializer_range = initializer_range
52
- self.rms_norm_eps = rms_norm_eps
53
- self.use_cache = use_cache
54
- self.rope_theta = rope_theta
55
- self.rope_scaling = rope_scaling
56
- self._rope_scaling_validation()
57
- self.attention_bias = attention_bias
58
- self.attention_dropout = attention_dropout
59
- self.model_parallel_size = model_parallel_size
60
- self.swin_norm = swin_norm
61
-
62
- if vq_config is None:
63
- vq_config = {}
64
-
65
- self.vq_config = ChameleonVQVAEConfig(**vq_config)
66
-
67
- self.vocabulary_map = vocabulary_map
68
-
69
- super().__init__(
70
- pad_token_id=pad_token_id,
71
- bos_token_id=bos_token_id,
72
- eos_token_id=eos_token_id,
73
- tie_word_embeddings=tie_word_embeddings,
74
- **kwargs,
75
- )
76
-
77
- def _rope_scaling_validation(self):
78
- """
79
- Validate the `rope_scaling` configuration.
80
- """
81
- if self.rope_scaling is None:
82
- return
83
-
84
- if not isinstance(self.rope_scaling,
85
- dict) or len(self.rope_scaling) != 2:
86
- raise ValueError(
87
- "`rope_scaling` must be a dictionary with with two fields, "
88
- f"`type` and `factor`, got {self.rope_scaling}")
89
- rope_scaling_type = self.rope_scaling.get("type", None)
90
- rope_scaling_factor = self.rope_scaling.get("factor", None)
91
- if rope_scaling_type is None or rope_scaling_type not in [
92
- "linear", "dynamic"
93
- ]:
94
- raise ValueError(
95
- "`rope_scaling`'s type field must be one of ['linear', "
96
- f"'dynamic'], got {rope_scaling_type}")
97
- if rope_scaling_factor is None or not isinstance(
98
- rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
99
- raise ValueError(
100
- "`rope_scaling`'s factor field must be a float > 1, "
101
- f"got {rope_scaling_factor}")
102
-
103
-
104
-class ChameleonVQVAEConfig(PretrainedConfig):
105
-
106
- model_type = "chameleon_vqgan"
107
-
108
- def __init__(
109
- self,
110
- embed_dim: int = 256,
111
- num_embeddings: int = 8192,
112
- double_latent: bool = False,
113
- latent_channels: int = 256,
114
- resolution: int = 512,
115
- in_channels: int = 3,
116
- base_channels: int = 128,
117
- channel_multiplier: List[int] = [1, 1, 2, 2, 4], #noqa
118
- num_res_blocks: int = 2,
119
- attn_resolutions: Optional[List[int]] = None,
120
- dropout: float = 0.0,
121
- attn_type: str = "vanilla",
122
- initializer_range=0.02,
123
- **kwargs,
124
- ):
125
- super().__init__(**kwargs)
126
- self.embed_dim = embed_dim
127
- self.num_embeddings = num_embeddings
128
- self.double_latent = double_latent
129
- self.latent_channels = latent_channels
130
- self.resolution = resolution
131
- self.in_channels = in_channels
132
- self.base_channels = base_channels
133
- self.channel_multiplier = channel_multiplier
134
- self.num_res_blocks = num_res_blocks
135
- self.attn_resolutions = attn_resolutions
136
- self.dropout = dropout
137
- self.attn_type = attn_type
138
- self.initializer_range = initializer_range