@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 94b5edeb53
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add qwen2 (#2495)

Junyang Lin committed Jan 23, 2024 at 06:34 UTC 94b5edeb5384ea2a46533a11dd5938b2c859bf5c
6 files changed +344 -2
README.md
+1
@@ -77,6 +77,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
77 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
78 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
79 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
80 +- Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
81 - StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
82 - Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
83
docs/source/models/supported_models.rst
+3
@@ -69,6 +69,9 @@ Alongside each architecture, we include some popular models that use it.
69 - Qwen
70 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
71 * - :code:`StableLMEpochForCausalLM`
72 + * - :code:`Qwen2ForCausalLM`
73 + - Qwen2
74 + - :code:`Qwen/Qwen2-7B-beta`, :code:`Qwen/Qwen-7B-Chat-beta`, etc.
75 - StableLM
76 - :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
77 * - :code:`YiForCausalLM`
requirements-rocm.txt
+1 -1
@@ -6,7 +6,7 @@ ray >= 2.5.1
6 sentencepiece # Required for LLaMA tokenizer.
7 numpy
8 tokenizers>=0.15.0
9 -transformers >= 4.36.0 # Required for Mixtral.
9 +transformers >= 4.37.0 # Required for Mixtral.
10 fastapi
11 uvicorn[standard]
12 pydantic >= 2.0 # Required for OpenAI server.
requirements.txt
+1 -1
@@ -4,7 +4,7 @@ ray >= 2.5.1
4 sentencepiece # Required for LLaMA tokenizer.
5 numpy
6 torch == 2.1.2
7 -transformers >= 4.36.0 # Required for Mixtral.
7 +transformers >= 4.37.0 # Required for Qwen2
8 xformers == 0.0.23.post1 # Required for CUDA 12.1.
9 fastapi
10 uvicorn[standard]
vllm/model_executor/models/__init__.py
+3
@@ -35,6 +35,7 @@ _MODELS = {
35 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
36 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
37 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38 + "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
39 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
40 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
41 "YiForCausalLM": ("yi", "YiForCausalLM")
@@ -46,6 +47,8 @@ _ROCM_UNSUPPORTED_MODELS = []
47 # Models partially supported by ROCm.
48 # Architecture -> Reason.
49 _ROCM_PARTIALLY_SUPPORTED_MODELS = {
50 + "Qwen2ForCausalLM":
51 + "Sliding window attention is not yet supported in ROCm's flash attention",
52 "MistralForCausalLM":
53 "Sliding window attention is not yet supported in ROCm's flash attention",
54 "MixtralForCausalLM":
vllm/model_executor/models/qwen2.py new
+335
@@ -0,0 +1,335 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py
4 +# Copyright 2024 The Qwen team and the HuggingFace Inc. team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Mistral model compatible with HuggingFace weights."""
24 +from typing import List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from transformers import Qwen2Config
29 +
30 +from vllm.model_executor.input_metadata import InputMetadata
31 +from vllm.model_executor.layers.activation import SiluAndMul
32 +from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.layernorm import RMSNorm
34 +from vllm.model_executor.layers.linear import (LinearMethodBase,
35 + MergedColumnParallelLinear,
36 + QKVParallelLinear,
37 + RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 +from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.vocab_parallel_embedding import (
41 + VocabParallelEmbedding, ParallelLMHead)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 +from vllm.sequence import SamplerOutput
48 +
49 +KVCache = Tuple[torch.Tensor, torch.Tensor]
50 +
51 +
52 +class Qwen2MLP(nn.Module):
53 +
54 + def __init__(
55 + self,
56 + hidden_size: int,
57 + intermediate_size: int,
58 + hidden_act: str,
59 + linear_method: Optional[LinearMethodBase] = None,
60 + ) -> None:
61 + super().__init__()
62 + self.gate_up_proj = MergedColumnParallelLinear(
63 + hidden_size, [intermediate_size] * 2,
64 + bias=False,
65 + linear_method=linear_method)
66 + self.down_proj = RowParallelLinear(intermediate_size,
67 + hidden_size,
68 + bias=False,
69 + linear_method=linear_method)
70 + if hidden_act != "silu":
71 + raise ValueError(f"Unsupported activation: {hidden_act}. "
72 + "Only silu is supported for now.")
73 + self.act_fn = SiluAndMul()
74 +
75 + def forward(self, x):
76 + gate_up, _ = self.gate_up_proj(x)
77 + x = self.act_fn(gate_up)
78 + x, _ = self.down_proj(x)
79 + return x
80 +
81 +
82 +class Qwen2Attention(nn.Module):
83 +
84 + def __init__(self,
85 + hidden_size: int,
86 + num_heads: int,
87 + num_kv_heads: int,
88 + max_position: int = 4096 * 32,
89 + rope_theta: float = 10000,
90 + use_sliding_window: bool = False,
91 + linear_method: Optional[LinearMethodBase] = None,
92 + sliding_window: Optional[int] = None) -> None:
93 + super().__init__()
94 + self.hidden_size = hidden_size
95 + tp_size = get_tensor_model_parallel_world_size()
96 + self.total_num_heads = num_heads
97 + assert self.total_num_heads % tp_size == 0
98 + self.num_heads = self.total_num_heads // tp_size
99 + self.total_num_kv_heads = num_kv_heads
100 + if self.total_num_kv_heads >= tp_size:
101 + # Number of KV heads is greater than TP size, so we partition
102 + # the KV heads across multiple tensor parallel GPUs.
103 + assert self.total_num_kv_heads % tp_size == 0
104 + else:
105 + # Number of KV heads is less than TP size, so we replicate
106 + # the KV heads across multiple tensor parallel GPUs.
107 + assert tp_size % self.total_num_kv_heads == 0
108 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109 + self.head_dim = hidden_size // self.total_num_heads
110 + self.q_size = self.num_heads * self.head_dim
111 + self.kv_size = self.num_kv_heads * self.head_dim
112 + self.scaling = self.head_dim**-0.5
113 + self.rope_theta = rope_theta
114 + self.sliding_window = sliding_window if use_sliding_window else None
115 +
116 + self.qkv_proj = QKVParallelLinear(
117 + hidden_size,
118 + self.head_dim,
119 + self.total_num_heads,
120 + self.total_num_kv_heads,
121 + bias=True,
122 + linear_method=linear_method,
123 + )
124 + self.o_proj = RowParallelLinear(
125 + self.total_num_heads * self.head_dim,
126 + hidden_size,
127 + bias=False,
128 + linear_method=linear_method,
129 + )
130 +
131 + self.rotary_emb = get_rope(
132 + self.head_dim,
133 + rotary_dim=self.head_dim,
134 + max_position=max_position,
135 + base=self.rope_theta,
136 + )
137 + self.attn = PagedAttention(self.num_heads,
138 + self.head_dim,
139 + self.scaling,
140 + num_kv_heads=self.num_kv_heads,
141 + sliding_window=self.sliding_window)
142 +
143 + def forward(
144 + self,
145 + positions: torch.Tensor,
146 + hidden_states: torch.Tensor,
147 + kv_cache: KVCache,
148 + input_metadata: InputMetadata,
149 + ) -> torch.Tensor:
150 + qkv, _ = self.qkv_proj(hidden_states)
151 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 + q, k = self.rotary_emb(positions, q, k)
153 + k_cache, v_cache = kv_cache
154 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 + output, _ = self.o_proj(attn_output)
156 + return output
157 +
158 +
159 +class Qwen2DecoderLayer(nn.Module):
160 +
161 + def __init__(
162 + self,
163 + config: Qwen2Config,
164 + layer_idx: int,
165 + linear_method: Optional[LinearMethodBase] = None,
166 + ) -> None:
167 + super().__init__()
168 + self.hidden_size = config.hidden_size
169 + # Requires transformers > 4.32.0
170 + rope_theta = getattr(config, "rope_theta", 1000000)
171 + use_sliding_window = config.use_sliding_window and layer_idx < config.max_window_layers
172 + self.self_attn = Qwen2Attention(
173 + hidden_size=self.hidden_size,
174 + num_heads=config.num_attention_heads,
175 + max_position=config.max_position_embeddings,
176 + num_kv_heads=config.num_key_value_heads,
177 + rope_theta=rope_theta,
178 + use_sliding_window=use_sliding_window,
179 + linear_method=linear_method,
180 + sliding_window=config.sliding_window)
181 + self.mlp = Qwen2MLP(
182 + hidden_size=self.hidden_size,
183 + intermediate_size=config.intermediate_size,
184 + hidden_act=config.hidden_act,
185 + linear_method=linear_method,
186 + )
187 + self.input_layernorm = RMSNorm(config.hidden_size,
188 + eps=config.rms_norm_eps)
189 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
190 + eps=config.rms_norm_eps)
191 +
192 + def forward(
193 + self,
194 + positions: torch.Tensor,
195 + hidden_states: torch.Tensor,
196 + kv_cache: KVCache,
197 + input_metadata: InputMetadata,
198 + residual: Optional[torch.Tensor],
199 + ) -> Tuple[torch.Tensor, torch.Tensor]:
200 + # Self Attention
201 + if residual is None:
202 + residual = hidden_states
203 + hidden_states = self.input_layernorm(hidden_states)
204 + else:
205 + hidden_states, residual = self.input_layernorm(
206 + hidden_states, residual)
207 + hidden_states = self.self_attn(
208 + positions=positions,
209 + hidden_states=hidden_states,
210 + kv_cache=kv_cache,
211 + input_metadata=input_metadata,
212 + )
213 +
214 + # Fully Connected
215 + hidden_states, residual = self.post_attention_layernorm(
216 + hidden_states, residual)
217 + hidden_states = self.mlp(hidden_states)
218 + return hidden_states, residual
219 +
220 +
221 +class Qwen2Model(nn.Module):
222 +
223 + def __init__(
224 + self,
225 + config: Qwen2Config,
226 + linear_method: Optional[LinearMethodBase] = None,
227 + ) -> None:
228 + super().__init__()
229 + self.config = config
230 + self.padding_idx = config.pad_token_id
231 + self.vocab_size = config.vocab_size
232 +
233 + self.embed_tokens = VocabParallelEmbedding(
234 + config.vocab_size,
235 + config.hidden_size,
236 + )
237 + self.layers = nn.ModuleList([
238 + Qwen2DecoderLayer(config, layer_idx, linear_method)
239 + for layer_idx in range(config.num_hidden_layers)
240 + ])
241 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
242 +
243 + def forward(
244 + self,
245 + input_ids: torch.Tensor,
246 + positions: torch.Tensor,
247 + kv_caches: List[KVCache],
248 + input_metadata: InputMetadata,
249 + ) -> torch.Tensor:
250 + hidden_states = self.embed_tokens(input_ids)
251 + residual = None
252 + for i in range(len(self.layers)):
253 + layer = self.layers[i]
254 + hidden_states, residual = layer(
255 + positions,
256 + hidden_states,
257 + kv_caches[i],
258 + input_metadata,
259 + residual,
260 + )
261 + hidden_states, _ = self.norm(hidden_states, residual)
262 + return hidden_states
263 +
264 +
265 +class Qwen2ForCausalLM(nn.Module):
266 +
267 + def __init__(
268 + self,
269 + config: Qwen2Config,
270 + linear_method: Optional[LinearMethodBase] = None,
271 + ) -> None:
272 + super().__init__()
273 + self.config = config
274 + self.linear_method = linear_method
275 + self.model = Qwen2Model(config, linear_method)
276 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
277 + self.sampler = Sampler(config.vocab_size)
278 +
279 + def forward(
280 + self,
281 + input_ids: torch.Tensor,
282 + positions: torch.Tensor,
283 + kv_caches: List[KVCache],
284 + input_metadata: InputMetadata,
285 + ) -> torch.Tensor:
286 + hidden_states = self.model(input_ids, positions, kv_caches,
287 + input_metadata)
288 + return hidden_states
289 +
290 + def sample(
291 + self,
292 + hidden_states: torch.Tensor,
293 + sampling_metadata: SamplingMetadata,
294 + ) -> Optional[SamplerOutput]:
295 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296 + sampling_metadata)
297 + return next_tokens
298 +
299 + def load_weights(self,
300 + model_name_or_path: str,
301 + cache_dir: Optional[str] = None,
302 + load_format: str = "auto",
303 + revision: Optional[str] = None):
304 + stacked_params_mapping = [
305 + # (param_name, shard_name, shard_id)
306 + ("qkv_proj", "q_proj", "q"),
307 + ("qkv_proj", "k_proj", "k"),
308 + ("qkv_proj", "v_proj", "v"),
309 + ("gate_up_proj", "gate_proj", 0),
310 + ("gate_up_proj", "up_proj", 1),
311 + ]
312 + params_dict = dict(self.named_parameters())
313 + for name, loaded_weight in hf_model_weights_iterator(
314 + model_name_or_path, cache_dir, load_format, revision):
315 + if "rotary_emb.inv_freq" in name:
316 + continue
317 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
318 + if weight_name not in name:
319 + continue
320 + name = name.replace(weight_name, param_name)
321 + # Skip loading extra bias for GPTQ models.
322 + if name.endswith(".bias") and name not in params_dict:
323 + continue
324 + param = params_dict[name]
325 + weight_loader = param.weight_loader
326 + weight_loader(param, loaded_weight, shard_id)
327 + break
328 + else:
329 + # Skip loading extra bias for GPTQ models.
330 + if name.endswith(".bias") and name not in params_dict:
331 + continue
332 + param = params_dict[name]
333 + weight_loader = getattr(param, "weight_loader",
334 + default_weight_loader)
335 + weight_loader(param, loaded_weight)