A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend] Dynamic RoPE scaling (#4638)
sasha0552 committed
May 22, 2024 at 05:32 UTC
9b9a10d6cb89f18e054daa66f25cb8f17c723b2c
5 files changed
+89
-12
tests/test_config.py
+55
-1
@@ -36,4 +36,58 @@ def test_get_sliding_window():
36
assert mistral_model_config.get_sliding_window() is None
37
38
mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
39
- assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
\ No newline at end of file
39
+ assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
40
+
41
+
42
+def test_rope_scaling():
43
+ TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
44
+ LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
45
+
46
+ llama_model_config = ModelConfig(
47
+ "meta-llama/Meta-Llama-3-8B-Instruct",
48
+ "meta-llama/Meta-Llama-3-8B-Instruct",
49
+ tokenizer_mode="auto",
50
+ trust_remote_code=False,
51
+ dtype="float16",
52
+ seed=0,
53
+ )
54
+ assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
55
+ assert llama_model_config.max_model_len == 8192
56
+
57
+ llama_model_config = ModelConfig(
58
+ "meta-llama/Meta-Llama-3-8B-Instruct",
59
+ "meta-llama/Meta-Llama-3-8B-Instruct",
60
+ tokenizer_mode="auto",
61
+ trust_remote_code=False,
62
+ dtype="float16",
63
+ seed=0,
64
+ rope_scaling=TEST_ROPE_SCALING,
65
+ )
66
+ assert getattr(llama_model_config.hf_config, "rope_scaling",
67
+ None) == TEST_ROPE_SCALING
68
+ assert llama_model_config.max_model_len == 16384
69
+
70
+ longchat_model_config = ModelConfig(
71
+ "lmsys/longchat-13b-16k",
72
+ "lmsys/longchat-13b-16k",
73
+ tokenizer_mode="auto",
74
+ trust_remote_code=False,
75
+ dtype="float16",
76
+ seed=0,
77
+ )
78
+ assert getattr(longchat_model_config.hf_config, "rope_scaling",
79
+ None) == LONGCHAT_ROPE_SCALING
80
+ assert longchat_model_config.max_model_len == 16384
81
+
82
+ longchat_model_config = ModelConfig(
83
+ "lmsys/longchat-13b-16k",
84
+ "lmsys/longchat-13b-16k",
85
+ tokenizer_mode="auto",
86
+ trust_remote_code=False,
87
+ dtype="float16",
88
+ seed=0,
89
+ rope_scaling=TEST_ROPE_SCALING,
90
+ )
91
+ assert getattr(longchat_model_config.hf_config, "rope_scaling",
92
+ None) == TEST_ROPE_SCALING
93
+ assert longchat_model_config.max_model_len == 4096
vllm/config.py
+6
-1
@@ -45,6 +45,9 @@ class ModelConfig:
45
code_revision: The specific revision to use for the model code on
46
Hugging Face Hub. It can be a branch name, a tag name, or a
47
commit id. If unspecified, will use the default version.
48
+ rope_scaling: Dictionary containing the scaling configuration for the
49
+ RoPE embeddings. When using this flag, don't update
50
+ `max_position_embeddings` to the expected new maximum.
51
tokenizer_revision: The specific tokenizer version to use. It can be a
52
branch name, a tag name, or a commit id. If unspecified, will use
53
the default version.
@@ -84,6 +87,7 @@ class ModelConfig:
87
seed: int,
88
revision: Optional[str] = None,
89
code_revision: Optional[str] = None,
90
+ rope_scaling: Optional[dict] = None,
91
tokenizer_revision: Optional[str] = None,
92
max_model_len: Optional[int] = None,
93
quantization: Optional[str] = None,
@@ -102,6 +106,7 @@ class ModelConfig:
106
self.seed = seed
107
self.revision = revision
108
self.code_revision = code_revision
109
+ self.rope_scaling = rope_scaling
110
self.tokenizer_revision = tokenizer_revision
111
self.quantization = quantization
112
self.quantization_param_path = quantization_param_path
@@ -116,7 +121,7 @@ class ModelConfig:
121
self.skip_tokenizer_init = skip_tokenizer_init
122
123
self.hf_config = get_config(self.model, trust_remote_code, revision,
119
- code_revision)
124
+ code_revision, rope_scaling)
125
self.hf_text_config = get_hf_text_config(self.hf_config)
126
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
127
self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
vllm/engine/arg_utils.py
+13
-5
@@ -1,5 +1,6 @@
1
import argparse
2
import dataclasses
3
+import json
4
from dataclasses import dataclass
5
from typing import List, Optional, Tuple, Union
6
@@ -49,6 +50,7 @@ class EngineArgs:
50
disable_log_stats: bool = False
51
revision: Optional[str] = None
52
code_revision: Optional[str] = None
53
+ rope_scaling: Optional[dict] = None
54
tokenizer_revision: Optional[str] = None
55
quantization: Optional[str] = None
56
enforce_eager: bool = False
@@ -330,6 +332,11 @@ class EngineArgs:
332
'None, we assume the model weights are not '
333
'quantized and use `dtype` to determine the data '
334
'type of the weights.')
335
+ parser.add_argument('--rope-scaling',
336
+ default=None,
337
+ type=json.loads,
338
+ help='RoPE scaling configuration in JSON format. '
339
+ 'For example, {"type":"dynamic","factor":2.0}')
340
parser.add_argument('--enforce-eager',
341
action='store_true',
342
help='Always use eager-mode PyTorch. If False, '
@@ -548,11 +555,12 @@ class EngineArgs:
555
model_config = ModelConfig(
556
self.model, self.tokenizer, self.tokenizer_mode,
557
self.trust_remote_code, self.dtype, self.seed, self.revision,
551
- self.code_revision, self.tokenizer_revision, self.max_model_len,
552
- self.quantization, self.quantization_param_path,
553
- self.enforce_eager, self.max_context_len_to_capture,
554
- self.max_seq_len_to_capture, self.max_logprobs,
555
- self.skip_tokenizer_init, self.served_model_name)
558
+ self.code_revision, self.rope_scaling, self.tokenizer_revision,
559
+ self.max_model_len, self.quantization,
560
+ self.quantization_param_path, self.enforce_eager,
561
+ self.max_context_len_to_capture, self.max_seq_len_to_capture,
562
+ self.max_logprobs, self.skip_tokenizer_init,
563
+ self.served_model_name)
564
cache_config = CacheConfig(self.block_size,
565
self.gpu_memory_utilization,
566
self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+6
-4
@@ -104,10 +104,11 @@ class LLMEngine:
104
"Initializing an LLM engine (v%s) with config: "
105
"model=%r, speculative_config=%r, tokenizer=%r, "
106
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
107
- "tokenizer_revision=%s, trust_remote_code=%s, dtype=%s, "
108
- "max_seq_len=%d, download_dir=%r, load_format=%s, "
109
- "tensor_parallel_size=%d, disable_custom_all_reduce=%s, "
110
- "quantization=%s, enforce_eager=%s, kv_cache_dtype=%s, "
107
+ "rope_scaling=%r, tokenizer_revision=%s, "
108
+ "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
109
+ "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
110
+ "disable_custom_all_reduce=%s, quantization=%s, "
111
+ "enforce_eager=%s, kv_cache_dtype=%s, "
112
"quantization_param_path=%s, device_config=%s, "
113
"decoding_config=%r, seed=%d, served_model_name=%s)",
114
vllm.__version__,
@@ -117,6 +118,7 @@ class LLMEngine:
118
model_config.skip_tokenizer_init,
119
model_config.tokenizer_mode,
120
model_config.revision,
121
+ model_config.rope_scaling,
122
model_config.tokenizer_revision,
123
model_config.trust_remote_code,
124
model_config.dtype,
vllm/transformers_utils/config.py
+9
-1
@@ -2,9 +2,12 @@ from typing import Dict, Optional
2
3
from transformers import AutoConfig, PretrainedConfig
4
5
+from vllm.logger import init_logger
6
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
7
JAISConfig, MPTConfig, RWConfig)
8
9
+logger = init_logger(__name__)
10
+
11
_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
12
"chatglm": ChatGLMConfig,
13
"dbrx": DbrxConfig,
@@ -18,7 +21,8 @@ _CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
21
def get_config(model: str,
22
trust_remote_code: bool,
23
revision: Optional[str] = None,
21
- code_revision: Optional[str] = None) -> PretrainedConfig:
24
+ code_revision: Optional[str] = None,
25
+ rope_scaling: Optional[dict] = None) -> PretrainedConfig:
26
try:
27
config = AutoConfig.from_pretrained(
28
model,
@@ -41,6 +45,10 @@ def get_config(model: str,
45
config = config_class.from_pretrained(model,
46
revision=revision,
47
code_revision=code_revision)
48
+ if rope_scaling is not None:
49
+ logger.info("Updating rope_scaling from %r to %r",
50
+ getattr(config, "rope_scaling", None), rope_scaling)
51
+ config.update({"rope_scaling": rope_scaling})
52
return config
53
54