@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 054072bee5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Minor] Move RoPE selection logic to `get_rope` (#1633)

Woosuk Kwon committed Nov 12, 2023 at 16:04 UTC 054072bee534faa9fae53dd9c14e91873d76204a
2 files changed +47 -34
vllm/model_executor/layers/attention.py
+3 -33
@@ -10,9 +10,7 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10 from vllm import attention_ops
11 from vllm import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 -from vllm.model_executor.layers.rotary_embedding import (
14 - DynamicNTKScalingRotaryEmbedding, LinearScalingRotaryEmbedding,
15 - RotaryEmbedding, YaRNScalingRotaryEmbedding)
13 +from vllm.model_executor.layers.rotary_embedding import get_rope
14
15 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
16 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -319,36 +317,8 @@ class PagedAttentionWithRoPE(PagedAttention):
317 scale,
318 num_kv_heads,
319 sliding_window=sliding_window)
322 - if rope_scaling is None:
323 - self.rotary_emb = RotaryEmbedding(head_size, rotary_dim,
324 - max_position, base,
325 - is_neox_style)
326 - else:
327 - scaling_type = rope_scaling["type"]
328 - scaling_factor = rope_scaling["factor"]
329 - if scaling_type == "linear":
330 - self.rotary_emb = LinearScalingRotaryEmbedding(
331 - head_size, rotary_dim, max_position, base, is_neox_style,
332 - scaling_factor)
333 - elif scaling_type == "dynamic":
334 - self.rotary_emb = DynamicNTKScalingRotaryEmbedding(
335 - head_size, rotary_dim, max_position, base, is_neox_style,
336 - scaling_factor)
337 - elif scaling_type == "yarn":
338 - original_max_position = rope_scaling[
339 - "original_max_position_embeddings"]
340 - assert max_position == original_max_position * scaling_factor
341 - extra_kwargs = {
342 - k: v
343 - for k, v in rope_scaling.items()
344 - if k in ("extrapolation_factor", "attn_factor",
345 - "beta_fast", "beta_slow")
346 - }
347 - self.rotary_emb = YaRNScalingRotaryEmbedding(
348 - head_size, rotary_dim, original_max_position, base,
349 - is_neox_style, scaling_factor, **extra_kwargs)
350 - else:
351 - raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
320 + self.rotary_emb = get_rope(head_size, rotary_dim, max_position, base,
321 + is_neox_style, rope_scaling)
322
323 def forward(
324 self,
vllm/model_executor/layers/rotary_embedding.py
+44 -1
@@ -22,7 +22,7 @@
22 # limitations under the License.
23 """Rotary Positional Embeddings."""
24 import math
25 -from typing import Tuple, Union
25 +from typing import Any, Dict, Optional, Tuple, Union
26
27 import torch
28 import torch.nn as nn
@@ -271,3 +271,46 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
271 sin = (freqs.sin() * self.mscale)
272 cache = torch.cat((cos, sin), dim=-1)
273 return cache
274 +
275 +
276 +def get_rope(
277 + head_size: int,
278 + rotary_dim: int,
279 + max_position: int,
280 + base: int,
281 + is_neox_style: bool,
282 + rope_scaling: Optional[Dict[str, Any]],
283 +) -> RotaryEmbedding:
284 + if rope_scaling is None:
285 + rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
286 + is_neox_style)
287 + else:
288 + scaling_type = rope_scaling["type"]
289 + scaling_factor = rope_scaling["factor"]
290 + if scaling_type == "linear":
291 + rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
292 + max_position, base,
293 + is_neox_style,
294 + scaling_factor)
295 + elif scaling_type == "dynamic":
296 + rotary_emb = DynamicNTKScalingRotaryEmbedding(
297 + head_size, rotary_dim, max_position, base, is_neox_style,
298 + scaling_factor)
299 + elif scaling_type == "yarn":
300 + original_max_position = rope_scaling[
301 + "original_max_position_embeddings"]
302 + assert max_position == original_max_position * scaling_factor
303 + extra_kwargs = {
304 + k: v
305 + for k, v in rope_scaling.items()
306 + if k in ("extrapolation_factor", "attn_factor", "beta_fast",
307 + "beta_slow")
308 + }
309 + rotary_emb = YaRNScalingRotaryEmbedding(head_size, rotary_dim,
310 + original_max_position,
311 + base, is_neox_style,
312 + scaling_factor,
313 + **extra_kwargs)
314 + else:
315 + raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
316 + return rotary_emb