@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d130b573a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] add rope_scaling support for qwen2 (#4930)

HUANG Fei committed May 21, 2024 at 13:22 UTC d130b573a0162173002b97e2112c6c1c10d0ca8e
1 file changed +6 -2
vllm/model_executor/models/qwen2.py
+6 -2
@@ -89,7 +89,8 @@ class Qwen2Attention(nn.Module):
89 use_sliding_window: bool = False,
90 cache_config: Optional[CacheConfig] = None,
91 quant_config: Optional[QuantizationConfig] = None,
92 - sliding_window: Optional[int] = None) -> None:
92 + sliding_window: Optional[int] = None,
93 + rope_scaling: Optional[Tuple] = None) -> None:
94 super().__init__()
95 self.hidden_size = hidden_size
96 tp_size = get_tensor_model_parallel_world_size()
@@ -133,6 +134,7 @@ class Qwen2Attention(nn.Module):
134 rotary_dim=self.head_dim,
135 max_position=max_position,
136 base=self.rope_theta,
137 + rope_scaling=rope_scaling,
138 )
139 self.attn = Attention(self.num_heads,
140 self.head_dim,
@@ -169,6 +171,7 @@ class Qwen2DecoderLayer(nn.Module):
171 self.hidden_size = config.hidden_size
172 # Requires transformers > 4.32.0
173 rope_theta = getattr(config, "rope_theta", 1000000)
174 + rope_scaling = getattr(config, "rope_scaling", None)
175 use_sliding_window = (config.use_sliding_window
176 and layer_idx < config.max_window_layers)
177 self.self_attn = Qwen2Attention(
@@ -180,7 +183,8 @@ class Qwen2DecoderLayer(nn.Module):
183 use_sliding_window=use_sliding_window,
184 cache_config=cache_config,
185 quant_config=quant_config,
183 - sliding_window=config.sliding_window)
186 + sliding_window=config.sliding_window,
187 + rope_scaling=rope_scaling)
188 self.mlp = Qwen2MLP(
189 hidden_size=self.hidden_size,
190 intermediate_size=config.intermediate_size,