@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3302f0aef3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

rope_theta and max_position_embeddings from config (#1096)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: wnma3mz <wnma3mz@gmail.com>

Antoni Baum committed Sep 20, 2023 at 13:35 UTC 3302f0aef39c392321567ac1400101155e365a29
9 files changed +140 -62
vllm/config.py
+39 -31
@@ -57,7 +57,7 @@ class ModelConfig:
57 load_format: str,
58 dtype: str,
59 seed: int,
60 - revision: Optional[str],
60 + revision: Optional[str] = None,
61 max_model_len: Optional[int] = None,
62 quantization: Optional[str] = None,
63 ) -> None:
@@ -73,19 +73,11 @@ class ModelConfig:
73
74 self.hf_config = get_config(model, trust_remote_code, revision)
75 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
76 + self.max_model_len = _get_and_verify_max_len(self.hf_config,
77 + max_model_len)
78 self._verify_load_format()
79 self._verify_tokenizer_mode()
80 self._verify_quantization()
79 - self.max_model_len = None
80 - if max_model_len is not None:
81 - derived_max_model_len = self.get_max_model_len()
82 - if max_model_len > derived_max_model_len:
83 - logger.warning(
84 - f"User-specified max_model_len ({max_model_len}) is "
85 - f"greater than the derived max_model_len "
86 - f"({derived_max_model_len}). Make sure the value is "
87 - "correct and within the model context size.")
88 - self.max_model_len = max_model_len
81
82 def _verify_load_format(self) -> None:
83 load_format = self.load_format.lower()
@@ -168,26 +160,7 @@ class ModelConfig:
160 return total_num_attention_heads // parallel_config.tensor_parallel_size
161
162 def get_max_model_len(self) -> int:
171 - if self.max_model_len is not None:
172 - return self.max_model_len
173 - max_model_len = float("inf")
174 - possible_keys = [
175 - # OPT
176 - "max_position_embeddings",
177 - # GPT-2
178 - "n_positions",
179 - # MPT
180 - "max_seq_len",
181 - # Others
182 - "max_sequence_length",
183 - "max_seq_length",
184 - "seq_len",
185 - ]
186 - for key in possible_keys:
187 - max_len_key = getattr(self.hf_config, key, None)
188 - if max_len_key is not None:
189 - max_model_len = min(max_model_len, max_len_key)
190 - return max_model_len
163 + return self.max_model_len
164
165 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
166 total_num_hidden_layers = self.hf_config.num_hidden_layers
@@ -348,3 +321,38 @@ def _get_and_verify_dtype(
321 f"of at least 8.0. Your {gpu_name} GPU has compute capability "
322 f"{compute_capability[0]}.{compute_capability[1]}.")
323 return torch_dtype
324 +
325 +
326 +def _get_and_verify_max_len(
327 + hf_config: PretrainedConfig,
328 + max_model_len: Optional[int],
329 +) -> int:
330 + """Get and verify the model's maximum length."""
331 + derived_max_model_len = float("inf")
332 + possible_keys = [
333 + # OPT
334 + "max_position_embeddings",
335 + # GPT-2
336 + "n_positions",
337 + # MPT
338 + "max_seq_len",
339 + # Others
340 + "max_sequence_length",
341 + "max_seq_length",
342 + "seq_len",
343 + ]
344 + for key in possible_keys:
345 + max_len_key = getattr(hf_config, key, None)
346 + if max_len_key is not None:
347 + derived_max_model_len = min(derived_max_model_len, max_len_key)
348 +
349 + if max_model_len is None:
350 + max_model_len = derived_max_model_len
351 + elif max_model_len > derived_max_model_len:
352 + raise ValueError(
353 + f"User-specified max_model_len ({max_model_len}) is greater than "
354 + f"the derived max_model_len ({max_len_key}={derived_max_model_len}"
355 + " in model's config.json). This may lead to incorrect model "
356 + "outputs or CUDA errors. Make sure the value is correct and "
357 + "within the model context size.")
358 + return max_model_len
vllm/model_executor/models/aquila.py
+11
@@ -105,6 +105,8 @@ class AquilaAttention(nn.Module):
105 hidden_size: int,
106 num_heads: int,
107 num_kv_heads: int,
108 + rope_theta: float = 10000,
109 + max_position_embeddings: int = 8192,
110 ):
111 super().__init__()
112 self.hidden_size = hidden_size
@@ -119,6 +121,8 @@ class AquilaAttention(nn.Module):
121 self.q_size = self.num_heads * self.head_dim
122 self.kv_size = self.num_kv_heads * self.head_dim
123 self.scaling = self.head_dim**-0.5
124 + self.rope_theta = rope_theta
125 + self.max_position_embeddings = max_position_embeddings
126
127 self.qkv_proj = ColumnParallelLinear(
128 hidden_size,
@@ -140,6 +144,8 @@ class AquilaAttention(nn.Module):
144 self.head_dim,
145 self.scaling,
146 rotary_dim=self.head_dim,
147 + base=self.rope_theta,
148 + max_position=self.max_position_embeddings,
149 )
150
151 def forward(
@@ -164,10 +170,15 @@ class AquilaDecoderLayer(nn.Module):
170 def __init__(self, config: AquilaConfig):
171 super().__init__()
172 self.hidden_size = config.hidden_size
173 + rope_theta = getattr(config, "rope_theta", 10000)
174 + max_position_embeddings = getattr(config, "max_position_embeddings",
175 + 8192)
176 self.self_attn = AquilaAttention(
177 hidden_size=self.hidden_size,
178 num_heads=config.num_attention_heads,
179 num_kv_heads=config.num_attention_heads,
180 + rope_theta=rope_theta,
181 + max_position_embeddings=max_position_embeddings,
182 )
183 self.mlp = AquilaMLP(
184 hidden_size=self.hidden_size,
vllm/model_executor/models/baichuan.py
+16 -4
@@ -111,6 +111,8 @@ class BaiChuanAttention(nn.Module):
111 hidden_size: int,
112 num_heads: int,
113 position_embedding: str,
114 + rope_theta: float = 10000,
115 + max_position_embeddings: int = 8192,
116 ):
117 super().__init__()
118 self.hidden_size = hidden_size
@@ -122,6 +124,8 @@ class BaiChuanAttention(nn.Module):
124 tensor_model_parallel_world_size)
125 self.head_dim = hidden_size // self.total_num_heads
126 self.postion_embedding = position_embedding
127 + self.rope_theta = rope_theta
128 + self.max_position_embeddings = max_position_embeddings
129
130 # pylint: disable=invalid-name
131 self.W_pack = ColumnParallelLinear(
@@ -151,10 +155,13 @@ class BaiChuanAttention(nn.Module):
155 scaling, alibi_slopes)
156 else:
157 self.scaling = self.head_dim**-0.5
154 - self.attn = PagedAttentionWithRoPE(self.num_heads,
155 - self.head_dim,
156 - self.scaling,
157 - rotary_dim=self.head_dim)
158 + self.attn = PagedAttentionWithRoPE(
159 + self.num_heads,
160 + self.head_dim,
161 + self.scaling,
162 + rotary_dim=self.head_dim,
163 + base=self.rope_theta,
164 + max_position=self.max_position_embeddings)
165
166 def forward(
167 self,
@@ -183,10 +190,15 @@ class BaiChuanDecoderLayer(nn.Module):
190 def __init__(self, config: BaiChuanConfig, position_embedding: str):
191 super().__init__()
192 self.hidden_size = config.hidden_size
193 + rope_theta = getattr(config, "rope_theta", 10000)
194 + max_position_embeddings = getattr(config, "max_position_embeddings",
195 + 8192)
196 self.self_attn = BaiChuanAttention(
197 hidden_size=self.hidden_size,
198 num_heads=config.num_attention_heads,
199 position_embedding=position_embedding,
200 + rope_theta=rope_theta,
201 + max_position_embeddings=max_position_embeddings,
202 )
203 self.mlp = BaiChuanMLP(
204 hidden_size=self.hidden_size,
vllm/model_executor/models/falcon.py
+11 -6
@@ -161,12 +161,17 @@ class FalconAttention(nn.Module):
161 "Rotary and alibi are mutually exclusive.")
162
163 if self.use_rotary:
164 - # TODO(zhuohan): Pass in correct `max_position``
165 - self.attn = PagedAttentionWithRoPE(self.num_heads,
166 - self.head_dim,
167 - self.inv_norm_factor,
168 - rotary_dim=self.head_dim,
169 - num_kv_heads=self.num_kv_heads)
164 + rope_theta = getattr(config, "rope_theta", 10000)
165 + max_position_embeddings = getattr(config,
166 + "max_position_embeddings", 8192)
167 + self.attn = PagedAttentionWithRoPE(
168 + self.num_heads,
169 + self.head_dim,
170 + self.inv_norm_factor,
171 + base=rope_theta,
172 + max_position=max_position_embeddings,
173 + rotary_dim=self.head_dim,
174 + num_kv_heads=self.num_kv_heads)
175 elif self.use_alibi:
176 tp_rank = get_tensor_model_parallel_rank()
177 head_start = tp_rank * self.num_heads
vllm/model_executor/models/gpt_j.py
+11 -5
@@ -67,11 +67,17 @@ class GPTJAttention(nn.Module):
67 scaling = self.head_size**-0.5
68 assert getattr(config, "rotary", True)
69 assert config.rotary_dim % 2 == 0
70 - self.attn = PagedAttentionWithRoPE(self.num_heads,
71 - self.head_size,
72 - scaling,
73 - config.rotary_dim,
74 - is_neox_style=False)
70 + rope_theta = getattr(config, "rope_theta", 10000)
71 + max_position_embeddings = getattr(config, "max_position_embeddings",
72 + 8192)
73 + self.attn = PagedAttentionWithRoPE(
74 + self.num_heads,
75 + self.head_size,
76 + scaling,
77 + config.rotary_dim,
78 + base=rope_theta,
79 + max_position=max_position_embeddings,
80 + is_neox_style=False)
81 self.warmup = False
82
83 def forward(
vllm/model_executor/models/gpt_neox.py
+10 -2
@@ -68,8 +68,16 @@ class GPTNeoXAttention(nn.Module):
68 scaling = self.head_size**-0.5
69 rotary_dim = int(self.head_size * config.rotary_pct)
70 assert rotary_dim % 2 == 0
71 - self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
72 - scaling, rotary_dim)
71 + rope_theta = getattr(config, "rope_theta", 10000)
72 + max_position_embeddings = getattr(config, "max_position_embeddings",
73 + 8192)
74 + self.attn = PagedAttentionWithRoPE(
75 + self.num_heads,
76 + self.head_size,
77 + scaling,
78 + rotary_dim,
79 + base=rope_theta,
80 + max_position=max_position_embeddings)
81
82 def forward(
83 self,
vllm/model_executor/models/internlm.py
+16 -4
@@ -59,6 +59,8 @@ class InternLMAttention(nn.Module):
59 self,
60 hidden_size: int,
61 num_heads: int,
62 + rope_theta: float = 10000,
63 + max_position_embeddings: int = 8192,
64 ):
65 super().__init__()
66 self.hidden_size = hidden_size
@@ -70,6 +72,8 @@ class InternLMAttention(nn.Module):
72 tensor_model_parallel_world_size)
73 self.head_dim = hidden_size // self.total_num_heads
74 self.scaling = self.head_dim**-0.5
75 + self.rope_theta = rope_theta
76 + self.max_position_embeddings = max_position_embeddings
77
78 self.qkv_proj = ColumnParallelLinear(
79 hidden_size,
@@ -85,10 +89,13 @@ class InternLMAttention(nn.Module):
89 input_is_parallel=True,
90 perform_initialization=False,
91 )
88 - self.attn = PagedAttentionWithRoPE(self.num_heads,
89 - self.head_dim,
90 - self.scaling,
91 - rotary_dim=self.head_dim)
92 + self.attn = PagedAttentionWithRoPE(
93 + self.num_heads,
94 + self.head_dim,
95 + self.scaling,
96 + base=self.rope_theta,
97 + max_position=self.max_position_embeddings,
98 + rotary_dim=self.head_dim)
99
100 def forward(
101 self,
@@ -112,9 +119,14 @@ class InternLMDecoderLayer(nn.Module):
119 def __init__(self, config: LlamaConfig):
120 super().__init__()
121 self.hidden_size = config.hidden_size
122 + rope_theta = getattr(config, "rope_theta", 10000)
123 + max_position_embeddings = getattr(config, "max_position_embeddings",
124 + 8192)
125 self.self_attn = InternLMAttention(
126 hidden_size=self.hidden_size,
127 num_heads=config.num_attention_heads,
128 + rope_theta=rope_theta,
129 + max_position_embeddings=max_position_embeddings,
130 )
131 self.mlp = InternLMMLP(
132 hidden_size=self.hidden_size,
vllm/model_executor/models/llama.py
+13 -6
@@ -92,6 +92,7 @@ class LlamaAttention(nn.Module):
92 num_heads: int,
93 num_kv_heads: int,
94 rope_theta: float = 10000,
95 + max_position_embeddings: int = 8192,
96 quant_config: Optional[QuantizationConfig] = None,
97 ) -> None:
98 super().__init__()
@@ -108,6 +109,7 @@ class LlamaAttention(nn.Module):
109 self.kv_size = self.num_kv_heads * self.head_dim
110 self.scaling = self.head_dim**-0.5
111 self.rope_theta = rope_theta
112 + self.max_position_embeddings = max_position_embeddings
113
114 self.qkv_proj = ParallelLinear.column(
115 hidden_size,
@@ -126,12 +128,14 @@ class LlamaAttention(nn.Module):
128 perform_initialization=False,
129 quant_config=quant_config,
130 )
129 - self.attn = PagedAttentionWithRoPE(self.num_heads,
130 - self.head_dim,
131 - self.scaling,
132 - base=self.rope_theta,
133 - rotary_dim=self.head_dim,
134 - num_kv_heads=self.num_kv_heads)
131 + self.attn = PagedAttentionWithRoPE(
132 + self.num_heads,
133 + self.head_dim,
134 + self.scaling,
135 + base=self.rope_theta,
136 + max_position=self.max_position_embeddings,
137 + rotary_dim=self.head_dim,
138 + num_kv_heads=self.num_kv_heads)
139
140 def forward(
141 self,
@@ -161,11 +165,14 @@ class LlamaDecoderLayer(nn.Module):
165 self.hidden_size = config.hidden_size
166 # Requires transformers > 4.32.0
167 rope_theta = getattr(config, "rope_theta", 10000)
168 + max_position_embeddings = getattr(config, "max_position_embeddings",
169 + 8192)
170 self.self_attn = LlamaAttention(
171 hidden_size=self.hidden_size,
172 num_heads=config.num_attention_heads,
173 num_kv_heads=config.num_key_value_heads,
174 rope_theta=rope_theta,
175 + max_position_embeddings=max_position_embeddings,
176 quant_config=quant_config,
177 )
178 self.mlp = LlamaMLP(
vllm/model_executor/models/qwen.py
+13 -4
@@ -76,8 +76,13 @@ class QWenMLP(nn.Module):
76
77 class QWenAttention(nn.Module):
78
79 - def __init__(self, hidden_size: int, num_heads: int,
80 - max_position_embeddings: int):
79 + def __init__(
80 + self,
81 + hidden_size: int,
82 + num_heads: int,
83 + max_position_embeddings: int,
84 + rope_theta: float = 10000,
85 + ):
86 super().__init__()
87 self.hidden_size = hidden_size
88 tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -109,6 +114,7 @@ class QWenAttention(nn.Module):
114 self.head_dim,
115 self.scaling,
116 rotary_dim=self.head_dim,
117 + base=rope_theta,
118 max_position=max_position_embeddings,
119 )
120
@@ -137,8 +143,11 @@ class QWenBlock(nn.Module):
143 super().__init__()
144 self.ln_1 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
145
140 - self.attn = QWenAttention(config.n_embd, config.num_attention_heads,
141 - config.max_position_embeddings)
146 + rope_theta = getattr(config, "rope_theta", 10000)
147 + self.attn = QWenAttention(config.n_embd,
148 + config.num_attention_heads,
149 + config.max_position_embeddings,
150 + rope_theta=rope_theta)
151
152 self.ln_2 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
153