@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 421e218b37
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Bump transformers to 4.43.2 (#6752)

Michael Goin committed Jul 24, 2024 at 16:22 UTC 421e218b37bd98b52bb3737c5aacc5a60fd460c0
2 files changed +29 -28
requirements-common.txt
+1 -1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6 requests
7 tqdm
8 py-cpuinfo
9 -transformers >= 4.43.1 # Required for Chameleon and Llama 3.1 hotfox.
9 +transformers >= 4.43.2 # Required for Chameleon and Llama 3.1 hotfox.
10 tokenizers >= 0.19.1 # Required for Llama 3.
11 fastapi
12 aiohttp
tests/test_config.py
+28 -27
@@ -64,8 +64,9 @@ def test_get_sliding_window():
64
65
66 def test_rope_customization():
67 - TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
67 + TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
68 TEST_ROPE_THETA = 16_000_000.0
69 + LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
70
71 llama_model_config = ModelConfig(
72 "meta-llama/Meta-Llama-3-8B-Instruct",
@@ -95,29 +96,29 @@ def test_rope_customization():
96 None) == TEST_ROPE_THETA
97 assert llama_model_config.max_model_len == 16384
98
98 - # TODO: add these back when the rope configs are fixed
99 - # LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
100 - # longchat_model_config = ModelConfig(
101 - # "lmsys/longchat-13b-16k",
102 - # "lmsys/longchat-13b-16k",
103 - # tokenizer_mode="auto",
104 - # trust_remote_code=False,
105 - # dtype="float16",
106 - # seed=0,
107 - # )
108 - # assert getattr(longchat_model_config.hf_config, "rope_scaling",
109 - # None) == LONGCHAT_ROPE_SCALING
110 - # assert longchat_model_config.max_model_len == 16384
111 -
112 - # longchat_model_config = ModelConfig(
113 - # "lmsys/longchat-13b-16k",
114 - # "lmsys/longchat-13b-16k",
115 - # tokenizer_mode="auto",
116 - # trust_remote_code=False,
117 - # dtype="float16",
118 - # seed=0,
119 - # rope_scaling=TEST_ROPE_SCALING,
120 - # )
121 - # assert getattr(longchat_model_config.hf_config, "rope_scaling",
122 - # None) == TEST_ROPE_SCALING
123 - # assert longchat_model_config.max_model_len == 4096
99 + longchat_model_config = ModelConfig(
100 + "lmsys/longchat-13b-16k",
101 + "lmsys/longchat-13b-16k",
102 + tokenizer_mode="auto",
103 + trust_remote_code=False,
104 + dtype="float16",
105 + seed=0,
106 + )
107 + # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
108 + assert all(
109 + longchat_model_config.hf_config.rope_scaling.get(key) == value
110 + for key, value in LONGCHAT_ROPE_SCALING.items())
111 + assert longchat_model_config.max_model_len == 16384
112 +
113 + longchat_model_config = ModelConfig(
114 + "lmsys/longchat-13b-16k",
115 + "lmsys/longchat-13b-16k",
116 + tokenizer_mode="auto",
117 + trust_remote_code=False,
118 + dtype="float16",
119 + seed=0,
120 + rope_scaling=TEST_ROPE_SCALING,
121 + )
122 + assert getattr(longchat_model_config.hf_config, "rope_scaling",
123 + None) == TEST_ROPE_SCALING
124 + assert longchat_model_config.max_model_len == 4096