@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b9a10d6cb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend] Dynamic RoPE scaling (#4638)

sasha0552 committed May 22, 2024 at 05:32 UTC 9b9a10d6cb89f18e054daa66f25cb8f17c723b2c
5 files changed +89 -12
tests/test_config.py
+55 -1
@@ -36,4 +36,58 @@ def test_get_sliding_window():
36 assert mistral_model_config.get_sliding_window() is None
37
38 mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
39 - assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
\ No newline at end of file
39 + assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
40 +
41 +
42 +def test_rope_scaling():
43 + TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
44 + LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
45 +
46 + llama_model_config = ModelConfig(
47 + "meta-llama/Meta-Llama-3-8B-Instruct",
48 + "meta-llama/Meta-Llama-3-8B-Instruct",
49 + tokenizer_mode="auto",
50 + trust_remote_code=False,
51 + dtype="float16",
52 + seed=0,
53 + )
54 + assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
55 + assert llama_model_config.max_model_len == 8192
56 +
57 + llama_model_config = ModelConfig(
58 + "meta-llama/Meta-Llama-3-8B-Instruct",
59 + "meta-llama/Meta-Llama-3-8B-Instruct",
60 + tokenizer_mode="auto",
61 + trust_remote_code=False,
62 + dtype="float16",
63 + seed=0,
64 + rope_scaling=TEST_ROPE_SCALING,
65 + )
66 + assert getattr(llama_model_config.hf_config, "rope_scaling",
67 + None) == TEST_ROPE_SCALING
68 + assert llama_model_config.max_model_len == 16384
69 +
70 + longchat_model_config = ModelConfig(
71 + "lmsys/longchat-13b-16k",
72 + "lmsys/longchat-13b-16k",
73 + tokenizer_mode="auto",
74 + trust_remote_code=False,
75 + dtype="float16",
76 + seed=0,
77 + )
78 + assert getattr(longchat_model_config.hf_config, "rope_scaling",
79 + None) == LONGCHAT_ROPE_SCALING
80 + assert longchat_model_config.max_model_len == 16384
81 +
82 + longchat_model_config = ModelConfig(
83 + "lmsys/longchat-13b-16k",
84 + "lmsys/longchat-13b-16k",
85 + tokenizer_mode="auto",
86 + trust_remote_code=False,
87 + dtype="float16",
88 + seed=0,
89 + rope_scaling=TEST_ROPE_SCALING,
90 + )
91 + assert getattr(longchat_model_config.hf_config, "rope_scaling",
92 + None) == TEST_ROPE_SCALING
93 + assert longchat_model_config.max_model_len == 4096
vllm/config.py
+6 -1
@@ -45,6 +45,9 @@ class ModelConfig:
45 code_revision: The specific revision to use for the model code on
46 Hugging Face Hub. It can be a branch name, a tag name, or a
47 commit id. If unspecified, will use the default version.
48 + rope_scaling: Dictionary containing the scaling configuration for the
49 + RoPE embeddings. When using this flag, don't update
50 + `max_position_embeddings` to the expected new maximum.
51 tokenizer_revision: The specific tokenizer version to use. It can be a
52 branch name, a tag name, or a commit id. If unspecified, will use
53 the default version.
@@ -84,6 +87,7 @@ class ModelConfig:
87 seed: int,
88 revision: Optional[str] = None,
89 code_revision: Optional[str] = None,
90 + rope_scaling: Optional[dict] = None,
91 tokenizer_revision: Optional[str] = None,
92 max_model_len: Optional[int] = None,
93 quantization: Optional[str] = None,
@@ -102,6 +106,7 @@ class ModelConfig:
106 self.seed = seed
107 self.revision = revision
108 self.code_revision = code_revision
109 + self.rope_scaling = rope_scaling
110 self.tokenizer_revision = tokenizer_revision
111 self.quantization = quantization
112 self.quantization_param_path = quantization_param_path
@@ -116,7 +121,7 @@ class ModelConfig:
121 self.skip_tokenizer_init = skip_tokenizer_init
122
123 self.hf_config = get_config(self.model, trust_remote_code, revision,
119 - code_revision)
124 + code_revision, rope_scaling)
125 self.hf_text_config = get_hf_text_config(self.hf_config)
126 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
127 self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
vllm/engine/arg_utils.py
+13 -5
@@ -1,5 +1,6 @@
1 import argparse
2 import dataclasses
3 +import json
4 from dataclasses import dataclass
5 from typing import List, Optional, Tuple, Union
6
@@ -49,6 +50,7 @@ class EngineArgs:
50 disable_log_stats: bool = False
51 revision: Optional[str] = None
52 code_revision: Optional[str] = None
53 + rope_scaling: Optional[dict] = None
54 tokenizer_revision: Optional[str] = None
55 quantization: Optional[str] = None
56 enforce_eager: bool = False
@@ -330,6 +332,11 @@ class EngineArgs:
332 'None, we assume the model weights are not '
333 'quantized and use `dtype` to determine the data '
334 'type of the weights.')
335 + parser.add_argument('--rope-scaling',
336 + default=None,
337 + type=json.loads,
338 + help='RoPE scaling configuration in JSON format. '
339 + 'For example, {"type":"dynamic","factor":2.0}')
340 parser.add_argument('--enforce-eager',
341 action='store_true',
342 help='Always use eager-mode PyTorch. If False, '
@@ -548,11 +555,12 @@ class EngineArgs:
555 model_config = ModelConfig(
556 self.model, self.tokenizer, self.tokenizer_mode,
557 self.trust_remote_code, self.dtype, self.seed, self.revision,
551 - self.code_revision, self.tokenizer_revision, self.max_model_len,
552 - self.quantization, self.quantization_param_path,
553 - self.enforce_eager, self.max_context_len_to_capture,
554 - self.max_seq_len_to_capture, self.max_logprobs,
555 - self.skip_tokenizer_init, self.served_model_name)
558 + self.code_revision, self.rope_scaling, self.tokenizer_revision,
559 + self.max_model_len, self.quantization,
560 + self.quantization_param_path, self.enforce_eager,
561 + self.max_context_len_to_capture, self.max_seq_len_to_capture,
562 + self.max_logprobs, self.skip_tokenizer_init,
563 + self.served_model_name)
564 cache_config = CacheConfig(self.block_size,
565 self.gpu_memory_utilization,
566 self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+6 -4
@@ -104,10 +104,11 @@ class LLMEngine:
104 "Initializing an LLM engine (v%s) with config: "
105 "model=%r, speculative_config=%r, tokenizer=%r, "
106 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
107 - "tokenizer_revision=%s, trust_remote_code=%s, dtype=%s, "
108 - "max_seq_len=%d, download_dir=%r, load_format=%s, "
109 - "tensor_parallel_size=%d, disable_custom_all_reduce=%s, "
110 - "quantization=%s, enforce_eager=%s, kv_cache_dtype=%s, "
107 + "rope_scaling=%r, tokenizer_revision=%s, "
108 + "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
109 + "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
110 + "disable_custom_all_reduce=%s, quantization=%s, "
111 + "enforce_eager=%s, kv_cache_dtype=%s, "
112 "quantization_param_path=%s, device_config=%s, "
113 "decoding_config=%r, seed=%d, served_model_name=%s)",
114 vllm.__version__,
@@ -117,6 +118,7 @@ class LLMEngine:
118 model_config.skip_tokenizer_init,
119 model_config.tokenizer_mode,
120 model_config.revision,
121 + model_config.rope_scaling,
122 model_config.tokenizer_revision,
123 model_config.trust_remote_code,
124 model_config.dtype,
vllm/transformers_utils/config.py
+9 -1
@@ -2,9 +2,12 @@ from typing import Dict, Optional
2
3 from transformers import AutoConfig, PretrainedConfig
4
5 +from vllm.logger import init_logger
6 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
7 JAISConfig, MPTConfig, RWConfig)
8
9 +logger = init_logger(__name__)
10 +
11 _CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
12 "chatglm": ChatGLMConfig,
13 "dbrx": DbrxConfig,
@@ -18,7 +21,8 @@ _CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
21 def get_config(model: str,
22 trust_remote_code: bool,
23 revision: Optional[str] = None,
21 - code_revision: Optional[str] = None) -> PretrainedConfig:
24 + code_revision: Optional[str] = None,
25 + rope_scaling: Optional[dict] = None) -> PretrainedConfig:
26 try:
27 config = AutoConfig.from_pretrained(
28 model,
@@ -41,6 +45,10 @@ def get_config(model: str,
45 config = config_class.from_pretrained(model,
46 revision=revision,
47 code_revision=code_revision)
48 + if rope_scaling is not None:
49 + logger.info("Updating rope_scaling from %r to %r",
50 + getattr(config, "rope_scaling", None), rope_scaling)
51 + config.update({"rope_scaling": rope_scaling})
52 return config
53
54