@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63b2206ad0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Avoid multiple instantiations of the RoPE class (#1828)

Jee Li committed Nov 30, 2023 at 15:06 UTC 63b2206ad01499921428ba50c85a18c92772f26c
1 file changed +8
vllm/model_executor/layers/rotary_embedding.py
+8
@@ -272,6 +272,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
272 return cache
273
274
275 +_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
276 +
277 +
278 def get_rope(
279 head_size: int,
280 rotary_dim: int,
@@ -280,6 +283,10 @@ def get_rope(
283 is_neox_style: bool = True,
284 rope_scaling: Optional[Dict[str, Any]] = None,
285 ) -> RotaryEmbedding:
286 + key = (head_size, rotary_dim, max_position, base, is_neox_style,
287 + rope_scaling)
288 + if key in _ROPE_DICT:
289 + return _ROPE_DICT[key]
290 if rope_scaling is None:
291 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
292 is_neox_style)
@@ -312,4 +319,5 @@ def get_rope(
319 **extra_kwargs)
320 else:
321 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
322 + _ROPE_DICT[key] = rotary_emb
323 return rotary_emb