A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] In LongRoPE, decide short vs long based on max_model_len (#27431)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Matthew Bonanni committed
Oct 28, 2025 at 08:00 UTC
44b5ce956d3cf28841615a58c1c0873af87bcfe2
3 files changed
+39
-11
tests/entrypoints/openai/test_default_mm_loras.py
+1
-1
@@ -29,7 +29,7 @@ def multimodal_server(): # noqa: F811
29
"--dtype",
30
"half",
31
"--max-model-len",
32
- "12800",
32
+ "4096",
33
"--enforce-eager",
34
# lora config below
35
"--enable-lora",
vllm/config/model.py
+11
-1
@@ -2142,8 +2142,18 @@ def _get_and_verify_max_len(
2142
# If the user didn't specify `max_model_len`, then use that derived from
2143
# the model config as a default value.
2144
if max_model_len is None:
2145
- max_model_len = int(derived_max_model_len)
2145
+ # For LongRoPE, default to original_max_position_embeddings to avoid
2146
+ # performance degradation for shorter sequences
2147
+ if rope_scaling is not None and rope_scaling["rope_type"] == "longrope":
2148
+ max_model_len = int(
2149
+ getattr(
2150
+ hf_config, "original_max_position_embeddings", derived_max_model_len
2151
+ )
2152
+ )
2153
+ else:
2154
+ max_model_len = int(derived_max_model_len)
2155
max_model_len = current_platform.check_max_model_len(max_model_len)
2156
+
2157
# If the user specified a max length, make sure it is smaller than the
2158
# derived length from the HF model config.
2159
elif max_model_len > derived_max_model_len:
vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py
+27
-9
@@ -5,8 +5,13 @@ import math
5
import torch
6
import torch.nn as nn
7
8
+from vllm.config import get_current_vllm_config
9
+from vllm.logger import init_logger
10
+
11
from .common import rotate_neox
12
13
+logger = init_logger(__name__)
14
+
15
16
class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
17
"""Phi3 family of models scaled rotary embedding.
@@ -43,6 +48,22 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
48
self.short_factor = short_factor
49
self.long_factor = long_factor
50
51
+ # Force long factors if max_model_len (runtime max length) exceeds
52
+ # original_max_position_embeddings to prevent KV cache invalidation when
53
+ # sequences cross this threshold during generation
54
+ max_model_len = get_current_vllm_config().model_config.max_model_len
55
+ self.use_long_rope = max_model_len > original_max_position_embeddings
56
+ if self.use_long_rope:
57
+ logger.warning_once(
58
+ "Using LongRoPE scaling factors. This enables longer "
59
+ "contexts (%d tokens vs original %d tokens) at the cost of "
60
+ "some performance degradation for shorter sequences. If "
61
+ "this is not desired, set `max_model_len` to be at most %d.",
62
+ max_position_embeddings,
63
+ original_max_position_embeddings,
64
+ original_max_position_embeddings,
65
+ )
66
+
67
scale = self.max_position_embeddings / self.original_max_position_embeddings
68
if scale <= 1.0:
69
scaling_factor = 1.0
@@ -112,15 +133,12 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
133
query = query.view(*query.shape[:-1], -1, self.head_size)
134
key = key.view(*key.shape[:-1], -1, self.head_size)
135
115
- k = self.original_max_position_embeddings
116
- long_prompt_offset = (
117
- torch.any(positions > k).float() * torch.full_like(positions, k)
118
- ).long()
119
- idx = (
120
- torch.add(positions, long_prompt_offset)
121
- if long_prompt_offset is not None
122
- else positions
123
- )
136
+ if self.use_long_rope:
137
+ k = self.original_max_position_embeddings
138
+ long_prompt_offset = torch.full_like(positions, k).long()
139
+ idx = torch.add(positions, long_prompt_offset)
140
+ else:
141
+ idx = positions
142
idx = torch.add(idx, offsets) if offsets is not None else idx
143
cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
144