A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add rope_scaling to Qwen (#1210)
Qing committed
Sep 28, 2023 at 15:49 UTC
7bedab574891ccdefb02a438b8a28bedd75a3b97
1 file changed
+11
-10
vllm/model_executor/models/qwen.py
+11
-10
@@ -8,7 +8,7 @@
8
The input of the model is flattened to a 1D tensor of tokens. The model uses
9
InputMetadata to extract the original 2D shape of the input.
10
"""
11
-from typing import List, Optional, Tuple
11
+from typing import Any, Dict, List, Optional, Tuple
12
13
import torch
14
from torch import nn
@@ -76,13 +76,12 @@ class QWenMLP(nn.Module):
76
77
class QWenAttention(nn.Module):
78
79
- def __init__(
80
- self,
81
- hidden_size: int,
82
- num_heads: int,
83
- max_position_embeddings: int,
84
- rope_theta: float = 10000,
85
- ):
79
+ def __init__(self,
80
+ hidden_size: int,
81
+ num_heads: int,
82
+ max_position_embeddings: int,
83
+ rope_theta: float = 10000,
84
+ rope_scaling: Optional[Dict[str, Any]] = None):
85
super().__init__()
86
self.hidden_size = hidden_size
87
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -116,7 +115,7 @@ class QWenAttention(nn.Module):
115
rotary_dim=self.head_dim,
116
base=rope_theta,
117
max_position=max_position_embeddings,
119
- )
118
+ rope_scaling=rope_scaling)
119
120
def forward(
121
self,
@@ -144,10 +143,12 @@ class QWenBlock(nn.Module):
143
self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
144
145
rope_theta = getattr(config, "rope_theta", 10000)
146
+ rope_scaling = getattr(config, "rope_scaling", None)
147
self.attn = QWenAttention(config.hidden_size,
148
config.num_attention_heads,
149
config.max_position_embeddings,
150
- rope_theta=rope_theta)
150
+ rope_theta=rope_theta,
151
+ rope_scaling=rope_scaling)
152
153
self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
154