@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7bedab5748
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add rope_scaling to Qwen (#1210)

Qing committed Sep 28, 2023 at 15:49 UTC 7bedab574891ccdefb02a438b8a28bedd75a3b97
1 file changed +11 -10
vllm/model_executor/models/qwen.py
+11 -10
@@ -8,7 +8,7 @@
8 The input of the model is flattened to a 1D tensor of tokens. The model uses
9 InputMetadata to extract the original 2D shape of the input.
10 """
11 -from typing import List, Optional, Tuple
11 +from typing import Any, Dict, List, Optional, Tuple
12
13 import torch
14 from torch import nn
@@ -76,13 +76,12 @@ class QWenMLP(nn.Module):
76
77 class QWenAttention(nn.Module):
78
79 - def __init__(
80 - self,
81 - hidden_size: int,
82 - num_heads: int,
83 - max_position_embeddings: int,
84 - rope_theta: float = 10000,
85 - ):
79 + def __init__(self,
80 + hidden_size: int,
81 + num_heads: int,
82 + max_position_embeddings: int,
83 + rope_theta: float = 10000,
84 + rope_scaling: Optional[Dict[str, Any]] = None):
85 super().__init__()
86 self.hidden_size = hidden_size
87 tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -116,7 +115,7 @@ class QWenAttention(nn.Module):
115 rotary_dim=self.head_dim,
116 base=rope_theta,
117 max_position=max_position_embeddings,
119 - )
118 + rope_scaling=rope_scaling)
119
120 def forward(
121 self,
@@ -144,10 +143,12 @@ class QWenBlock(nn.Module):
143 self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
144
145 rope_theta = getattr(config, "rope_theta", 10000)
146 + rope_scaling = getattr(config, "rope_scaling", None)
147 self.attn = QWenAttention(config.hidden_size,
148 config.num_attention_heads,
149 config.max_position_embeddings,
150 - rope_theta=rope_theta)
150 + rope_theta=rope_theta,
151 + rope_scaling=rope_scaling)
152
153 self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
154