A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Bump transformers to 4.43.2 (#6752)
Michael Goin committed
Jul 24, 2024 at 16:22 UTC
421e218b37bd98b52bb3737c5aacc5a60fd460c0
2 files changed
+29
-28
requirements-common.txt
+1
-1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6
requests
7
tqdm
8
py-cpuinfo
9
-transformers >= 4.43.1 # Required for Chameleon and Llama 3.1 hotfox.
9
+transformers >= 4.43.2 # Required for Chameleon and Llama 3.1 hotfox.
10
tokenizers >= 0.19.1 # Required for Llama 3.
11
fastapi
12
aiohttp
tests/test_config.py
+28
-27
@@ -64,8 +64,9 @@ def test_get_sliding_window():
64
65
66
def test_rope_customization():
67
- TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
67
+ TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
68
TEST_ROPE_THETA = 16_000_000.0
69
+ LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
70
71
llama_model_config = ModelConfig(
72
"meta-llama/Meta-Llama-3-8B-Instruct",
@@ -95,29 +96,29 @@ def test_rope_customization():
96
None) == TEST_ROPE_THETA
97
assert llama_model_config.max_model_len == 16384
98
98
- # TODO: add these back when the rope configs are fixed
99
- # LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
100
- # longchat_model_config = ModelConfig(
101
- # "lmsys/longchat-13b-16k",
102
- # "lmsys/longchat-13b-16k",
103
- # tokenizer_mode="auto",
104
- # trust_remote_code=False,
105
- # dtype="float16",
106
- # seed=0,
107
- # )
108
- # assert getattr(longchat_model_config.hf_config, "rope_scaling",
109
- # None) == LONGCHAT_ROPE_SCALING
110
- # assert longchat_model_config.max_model_len == 16384
111
-
112
- # longchat_model_config = ModelConfig(
113
- # "lmsys/longchat-13b-16k",
114
- # "lmsys/longchat-13b-16k",
115
- # tokenizer_mode="auto",
116
- # trust_remote_code=False,
117
- # dtype="float16",
118
- # seed=0,
119
- # rope_scaling=TEST_ROPE_SCALING,
120
- # )
121
- # assert getattr(longchat_model_config.hf_config, "rope_scaling",
122
- # None) == TEST_ROPE_SCALING
123
- # assert longchat_model_config.max_model_len == 4096
99
+ longchat_model_config = ModelConfig(
100
+ "lmsys/longchat-13b-16k",
101
+ "lmsys/longchat-13b-16k",
102
+ tokenizer_mode="auto",
103
+ trust_remote_code=False,
104
+ dtype="float16",
105
+ seed=0,
106
+ )
107
+ # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
108
+ assert all(
109
+ longchat_model_config.hf_config.rope_scaling.get(key) == value
110
+ for key, value in LONGCHAT_ROPE_SCALING.items())
111
+ assert longchat_model_config.max_model_len == 16384
112
+
113
+ longchat_model_config = ModelConfig(
114
+ "lmsys/longchat-13b-16k",
115
+ "lmsys/longchat-13b-16k",
116
+ tokenizer_mode="auto",
117
+ trust_remote_code=False,
118
+ dtype="float16",
119
+ seed=0,
120
+ rope_scaling=TEST_ROPE_SCALING,
121
+ )
122
+ assert getattr(longchat_model_config.hf_config, "rope_scaling",
123
+ None) == TEST_ROPE_SCALING
124
+ assert longchat_model_config.max_model_len == 4096