A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] add rope_scaling support for qwen2 (#4930)
HUANG Fei committed
May 21, 2024 at 13:22 UTC
d130b573a0162173002b97e2112c6c1c10d0ca8e
1 file changed
+6
-2
vllm/model_executor/models/qwen2.py
+6
-2
@@ -89,7 +89,8 @@ class Qwen2Attention(nn.Module):
89
use_sliding_window: bool = False,
90
cache_config: Optional[CacheConfig] = None,
91
quant_config: Optional[QuantizationConfig] = None,
92
- sliding_window: Optional[int] = None) -> None:
92
+ sliding_window: Optional[int] = None,
93
+ rope_scaling: Optional[Tuple] = None) -> None:
94
super().__init__()
95
self.hidden_size = hidden_size
96
tp_size = get_tensor_model_parallel_world_size()
@@ -133,6 +134,7 @@ class Qwen2Attention(nn.Module):
134
rotary_dim=self.head_dim,
135
max_position=max_position,
136
base=self.rope_theta,
137
+ rope_scaling=rope_scaling,
138
)
139
self.attn = Attention(self.num_heads,
140
self.head_dim,
@@ -169,6 +171,7 @@ class Qwen2DecoderLayer(nn.Module):
171
self.hidden_size = config.hidden_size
172
# Requires transformers > 4.32.0
173
rope_theta = getattr(config, "rope_theta", 1000000)
174
+ rope_scaling = getattr(config, "rope_scaling", None)
175
use_sliding_window = (config.use_sliding_window
176
and layer_idx < config.max_window_layers)
177
self.self_attn = Qwen2Attention(
@@ -180,7 +183,8 @@ class Qwen2DecoderLayer(nn.Module):
183
use_sliding_window=use_sliding_window,
184
cache_config=cache_config,
185
quant_config=quant_config,
183
- sliding_window=config.sliding_window)
186
+ sliding_window=config.sliding_window,
187
+ rope_scaling=rope_scaling)
188
self.mlp = Qwen2MLP(
189
hidden_size=self.hidden_size,
190
intermediate_size=config.intermediate_size,