A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Avoid multiple instantiations of the RoPE class (#1828)
Jee Li committed
Nov 30, 2023 at 15:06 UTC
63b2206ad01499921428ba50c85a18c92772f26c
1 file changed
+8
vllm/model_executor/layers/rotary_embedding.py
+8
@@ -272,6 +272,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
272
return cache
273
274
275
+_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
276
+
277
+
278
def get_rope(
279
head_size: int,
280
rotary_dim: int,
@@ -280,6 +283,10 @@ def get_rope(
283
is_neox_style: bool = True,
284
rope_scaling: Optional[Dict[str, Any]] = None,
285
) -> RotaryEmbedding:
286
+ key = (head_size, rotary_dim, max_position, base, is_neox_style,
287
+ rope_scaling)
288
+ if key in _ROPE_DICT:
289
+ return _ROPE_DICT[key]
290
if rope_scaling is None:
291
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
292
is_neox_style)
@@ -312,4 +319,5 @@ def get_rope(
319
**extra_kwargs)
320
else:
321
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
322
+ _ROPE_DICT[key] = rotary_emb
323
return rotary_emb